<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="en-US">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=11"/>
<meta name="generator" content="Doxygen 1.12.0"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>NeuZephyr: D:/Users/Mgepahmge/Documents/C Program/NeuZephyr/src/OperationKernels.cu Source File</title>
<link rel="icon" href="NZ_logo2.png" type="image/x-icon" />
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="dynsections.js"></script>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="resize.js"></script>
<link href="doxygen.css" rel="stylesheet" type="text/css" />
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr id="projectrow">
  <td id="projectlogo"><img alt="Logo" src="NZ_logo2.png"/></td>
  <td id="projectalign">
   <div id="projectname">NeuZephyr
   </div>
   <div id="projectbrief">Simple DL Framework</div>
  </td>
 </tr>
 </tbody>
</table>
</div>
<!-- end header part -->
<!-- Generated by Doxygen 1.12.0 -->
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&amp;dn=expat.txt MIT */
$(function() { codefold.init(0); });
/* @license-end */
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li><a href="pages.html"><span>Related&#160;Pages</span></a></li>
      <li><a href="namespaces.html"><span>Namespaces</span></a></li>
      <li><a href="annotated.html"><span>Classes</span></a></li>
      <li class="current"><a href="files.html"><span>Files</span></a></li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="files.html"><span>File&#160;List</span></a></li>
    </ul>
  </div>
<script type="text/javascript">
/* @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&amp;dn=expat.txt MIT */
$(function(){ initResizable(false); });
/* @license-end */
</script>
<div id="nav-path" class="navpath">
  <ul>
<li class="navelem"><a class="el" href="dir_d522931ffa1371640980b621734a4381.html">Users</a></li><li class="navelem"><a class="el" href="dir_a7e6ee1ae3f772c9504a0b543f2027e2.html">Mgepahmge</a></li><li class="navelem"><a class="el" href="dir_e03f57e346cc4845a4c354a35630b169.html">Documents</a></li><li class="navelem"><a class="el" href="dir_231a0482af2b83c895f27ba7fe745141.html">C Program</a></li><li class="navelem"><a class="el" href="dir_0fa7fc3a0dfd304dbfc9dce9f6facfa2.html">NeuZephyr</a></li><li class="navelem"><a class="el" href="dir_25794e61537e3f33113e2168c9f8da60.html">src</a></li>  </ul>
</div>
</div><!-- top -->
<div id="doc-content">
<div class="header">
  <div class="headertitle"><div class="title">OperationKernels.cu</div></div>
</div><!--header-->
<div class="contents">
<div class="fragment"><div class="line"><a id="l00001" name="l00001"></a><span class="lineno">    1</span><span class="preprocessor">#include &quot;<a class="code" href="_operation_kernels_8cuh.html">NeuZephyr/OperationKernels.cuh</a>&quot;</span></div>
<div class="line"><a id="l00002" name="l00002"></a><span class="lineno">    2</span><span class="preprocessor">#include &quot;NeuZephyr/utils.cuh&quot;</span></div>
<div class="line"><a id="l00003" name="l00003"></a><span class="lineno">    3</span><span class="preprocessor">#include &quot;NeuZephyr/StreamManager.cuh&quot;</span></div>
<div class="line"><a id="l00004" name="l00004"></a><span class="lineno">    4</span><span class="preprocessor">#include &quot;NeuZephyr/Dimension.cuh&quot;</span></div>
<div class="line"><a id="l00005" name="l00005"></a><span class="lineno">    5</span><span class="preprocessor">#include &lt;cuda_runtime.h&gt;</span></div>
<div class="line"><a id="l00006" name="l00006"></a><span class="lineno">    6</span><span class="preprocessor">#include &lt;cuda_fp16.h&gt;</span></div>
<div class="line"><a id="l00007" name="l00007"></a><span class="lineno">    7</span><span class="preprocessor">#include &lt;mma.h&gt;</span></div>
<div class="line"><a id="l00008" name="l00008"></a><span class="lineno">    8</span> </div>
<div class="line"><a id="l00009" name="l00009"></a><span class="lineno">    9</span><span class="keyword">namespace </span><a class="code hl_namespace" href="namespacenz_1_1krnl.html">nz::krnl</a> {</div>
<div class="line"><a id="l00010" name="l00010"></a><span class="lineno">   10</span>    <span class="keyword">using namespace </span>cuStrm;</div>
<div class="line"><a id="l00011" name="l00011"></a><span class="lineno">   11</span> </div>
<div class="line"><a id="l00012" name="l00012"></a><span class="lineno">   12</span>    __global__ <span class="keywordtype">void</span> MatrixAddKernel(<span class="keywordtype">float</span>* c, <span class="keyword">const</span> <span class="keywordtype">float</span>* a, <span class="keyword">const</span> <span class="keywordtype">float</span>* b,</div>
<div class="line"><a id="l00013" name="l00013"></a><span class="lineno">   13</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l00014" name="l00014"></a><span class="lineno">   14</span>                                    <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_c,</div>
<div class="line"><a id="l00015" name="l00015"></a><span class="lineno">   15</span>                                    <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_a,</div>
<div class="line"><a id="l00016" name="l00016"></a><span class="lineno">   16</span>                                    <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_b) {</div>
<div class="line"><a id="l00017" name="l00017"></a><span class="lineno">   17</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00018" name="l00018"></a><span class="lineno">   18</span>        <span class="keywordtype">float</span>* c_m = c + offset_c;</div>
<div class="line"><a id="l00019" name="l00019"></a><span class="lineno">   19</span>        <span class="keyword">const</span> <span class="keywordtype">float</span>* a_m = a + offset_a;</div>
<div class="line"><a id="l00020" name="l00020"></a><span class="lineno">   20</span>        <span class="keyword">const</span> <span class="keywordtype">float</span>* b_m = b + offset_b;</div>
<div class="line"><a id="l00021" name="l00021"></a><span class="lineno">   21</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00022" name="l00022"></a><span class="lineno">   22</span>            c_m[idx] = a_m[idx] + b_m[idx];</div>
<div class="line"><a id="l00023" name="l00023"></a><span class="lineno">   23</span>        }</div>
<div class="line"><a id="l00024" name="l00024"></a><span class="lineno">   24</span>    }</div>
<div class="line"><a id="l00025" name="l00025"></a><span class="lineno">   25</span> </div>
<div class="foldopen" id="foldopen00026" data-start="{" data-end="}">
<div class="line"><a id="l00026" name="l00026"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a97cda6dfc6545efaee2b686eed9ae766">   26</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a97cda6dfc6545efaee2b686eed9ae766">MatrixAdd</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* a, <span class="keywordtype">float</span>* b, <span class="keywordtype">float</span>* c,</div>
<div class="line"><a id="l00027" name="l00027"></a><span class="lineno">   27</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_c, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_a, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_b) {</div>
<div class="line"><a id="l00028" name="l00028"></a><span class="lineno">   28</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(MatrixAddKernel, gridDim, blockDim, 0, c, a, b, n, offset_c, offset_a,</div>
<div class="line"><a id="l00029" name="l00029"></a><span class="lineno">   29</span>                                                offset_b);</div>
<div class="line"><a id="l00030" name="l00030"></a><span class="lineno">   30</span>    }</div>
</div>
<div class="line"><a id="l00031" name="l00031"></a><span class="lineno">   31</span> </div>
<div class="foldopen" id="foldopen00032" data-start="{" data-end="}">
<div class="line"><a id="l00032" name="l00032"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a5b29c405a1df9534430ad8682960ebb5">   32</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a97cda6dfc6545efaee2b686eed9ae766">MatrixAdd</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* a, <span class="keywordtype">float</span>* b, <span class="keywordtype">float</span>* c,</div>
<div class="line"><a id="l00033" name="l00033"></a><span class="lineno">   33</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_c, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_a,</div>
<div class="line"><a id="l00034" name="l00034"></a><span class="lineno">   34</span>                   <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_b) {</div>
<div class="line"><a id="l00035" name="l00035"></a><span class="lineno">   35</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitParallel(MatrixAddKernel, gridDim, blockDim, 0, c, a, b, offset_c,</div>
<div class="line"><a id="l00036" name="l00036"></a><span class="lineno">   36</span>                                                        offset_a, offset_b, n);</div>
<div class="line"><a id="l00037" name="l00037"></a><span class="lineno">   37</span>    }</div>
</div>
<div class="line"><a id="l00038" name="l00038"></a><span class="lineno">   38</span> </div>
<div class="line"><a id="l00039" name="l00039"></a><span class="lineno">   39</span>    __global__ <span class="keywordtype">void</span> MatrixSubKernel(<span class="keywordtype">float</span>* c, <span class="keyword">const</span> <span class="keywordtype">float</span>* a, <span class="keyword">const</span> <span class="keywordtype">float</span>* b,</div>
<div class="line"><a id="l00040" name="l00040"></a><span class="lineno">   40</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l00041" name="l00041"></a><span class="lineno">   41</span>                                    <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_c,</div>
<div class="line"><a id="l00042" name="l00042"></a><span class="lineno">   42</span>                                    <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_a,</div>
<div class="line"><a id="l00043" name="l00043"></a><span class="lineno">   43</span>                                    <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_b) {</div>
<div class="line"><a id="l00044" name="l00044"></a><span class="lineno">   44</span>        <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00045" name="l00045"></a><span class="lineno">   45</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00046" name="l00046"></a><span class="lineno">   46</span>            c[idx + offset_c] = a[idx + offset_a] - b[idx + offset_b];</div>
<div class="line"><a id="l00047" name="l00047"></a><span class="lineno">   47</span>        }</div>
<div class="line"><a id="l00048" name="l00048"></a><span class="lineno">   48</span>    }</div>
<div class="line"><a id="l00049" name="l00049"></a><span class="lineno">   49</span> </div>
<div class="foldopen" id="foldopen00050" data-start="{" data-end="}">
<div class="line"><a id="l00050" name="l00050"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#ad18a2b0efc0cdfc9cb861396ad4da53f">   50</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#ad18a2b0efc0cdfc9cb861396ad4da53f">MatrixSub</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* a, <span class="keywordtype">float</span>* b, <span class="keywordtype">float</span>* c,</div>
<div class="line"><a id="l00051" name="l00051"></a><span class="lineno">   51</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_c,</div>
<div class="line"><a id="l00052" name="l00052"></a><span class="lineno">   52</span>                   <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_a,</div>
<div class="line"><a id="l00053" name="l00053"></a><span class="lineno">   53</span>                   <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_b) {</div>
<div class="line"><a id="l00054" name="l00054"></a><span class="lineno">   54</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(MatrixSubKernel, gridDim, blockDim, 0, c, a, b, n, offset_c, offset_a,</div>
<div class="line"><a id="l00055" name="l00055"></a><span class="lineno">   55</span>                                                offset_b);</div>
<div class="line"><a id="l00056" name="l00056"></a><span class="lineno">   56</span>    }</div>
</div>
<div class="line"><a id="l00057" name="l00057"></a><span class="lineno">   57</span> </div>
<div class="foldopen" id="foldopen00058" data-start="{" data-end="}">
<div class="line"><a id="l00058" name="l00058"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a4ca041c74dc55e3ac9124b5fd39b985c">   58</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#ad18a2b0efc0cdfc9cb861396ad4da53f">MatrixSub</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* a, <span class="keywordtype">float</span>* b, <span class="keywordtype">float</span>* c,</div>
<div class="line"><a id="l00059" name="l00059"></a><span class="lineno">   59</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_c, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_a,</div>
<div class="line"><a id="l00060" name="l00060"></a><span class="lineno">   60</span>                   <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_b) {</div>
<div class="line"><a id="l00061" name="l00061"></a><span class="lineno">   61</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitParallel(MatrixSubKernel, gridDim, blockDim, 0, c, a, b, offset_c,</div>
<div class="line"><a id="l00062" name="l00062"></a><span class="lineno">   62</span>                                                        offset_a, offset_b, n);</div>
<div class="line"><a id="l00063" name="l00063"></a><span class="lineno">   63</span>    }</div>
</div>
<div class="line"><a id="l00064" name="l00064"></a><span class="lineno">   64</span> </div>
<div class="line"><a id="l00065" name="l00065"></a><span class="lineno">   65</span>    __global__ <span class="keywordtype">void</span> GeneralMatrixMulKernel(<span class="keywordtype">float</span>* C, <span class="keyword">const</span> <span class="keywordtype">float</span>* A, <span class="keyword">const</span> <span class="keywordtype">float</span>* B,</div>
<div class="line"><a id="l00066" name="l00066"></a><span class="lineno">   66</span>                                           <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> M,</div>
<div class="line"><a id="l00067" name="l00067"></a><span class="lineno">   67</span>                                           <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> N,</div>
<div class="line"><a id="l00068" name="l00068"></a><span class="lineno">   68</span>                                           <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> K,</div>
<div class="line"><a id="l00069" name="l00069"></a><span class="lineno">   69</span>                                           <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_c,</div>
<div class="line"><a id="l00070" name="l00070"></a><span class="lineno">   70</span>                                           <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_a,</div>
<div class="line"><a id="l00071" name="l00071"></a><span class="lineno">   71</span>                                           <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_b) {</div>
<div class="line"><a id="l00072" name="l00072"></a><span class="lineno">   72</span>        __shared__ <span class="keywordtype">float</span> As[TILE_SIZE][TILE_SIZE];</div>
<div class="line"><a id="l00073" name="l00073"></a><span class="lineno">   73</span>        __shared__ <span class="keywordtype">float</span> Bs[TILE_SIZE][TILE_SIZE];</div>
<div class="line"><a id="l00074" name="l00074"></a><span class="lineno">   74</span> </div>
<div class="line"><a id="l00075" name="l00075"></a><span class="lineno">   75</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> row = blockIdx.y * blockDim.y + threadIdx.y;</div>
<div class="line"><a id="l00076" name="l00076"></a><span class="lineno">   76</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> col = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00077" name="l00077"></a><span class="lineno">   77</span> </div>
<div class="line"><a id="l00078" name="l00078"></a><span class="lineno">   78</span>        <span class="keywordtype">float</span> sum = 0.0f;</div>
<div class="line"><a id="l00079" name="l00079"></a><span class="lineno">   79</span> </div>
<div class="line"><a id="l00080" name="l00080"></a><span class="lineno">   80</span>        <span class="keywordflow">for</span> (<span class="keywordtype">int</span> t = 0; t &lt; K; t += TILE_SIZE) {</div>
<div class="line"><a id="l00081" name="l00081"></a><span class="lineno">   81</span>            <span class="keywordflow">if</span> (row &lt; M &amp;&amp; t + threadIdx.x &lt; K)</div>
<div class="line"><a id="l00082" name="l00082"></a><span class="lineno">   82</span>                As[threadIdx.y][threadIdx.x] = A[row * K + t + threadIdx.x + offset_a];</div>
<div class="line"><a id="l00083" name="l00083"></a><span class="lineno">   83</span>            <span class="keywordflow">else</span></div>
<div class="line"><a id="l00084" name="l00084"></a><span class="lineno">   84</span>                As[threadIdx.y][threadIdx.x] = 0.0f;</div>
<div class="line"><a id="l00085" name="l00085"></a><span class="lineno">   85</span> </div>
<div class="line"><a id="l00086" name="l00086"></a><span class="lineno">   86</span>            <span class="keywordflow">if</span> (col &lt; N &amp;&amp; t + threadIdx.y &lt; K)</div>
<div class="line"><a id="l00087" name="l00087"></a><span class="lineno">   87</span>                Bs[threadIdx.y][threadIdx.x] = B[(t + threadIdx.y) * N + col + offset_b];</div>
<div class="line"><a id="l00088" name="l00088"></a><span class="lineno">   88</span>            <span class="keywordflow">else</span></div>
<div class="line"><a id="l00089" name="l00089"></a><span class="lineno">   89</span>                Bs[threadIdx.y][threadIdx.x] = 0.0f;</div>
<div class="line"><a id="l00090" name="l00090"></a><span class="lineno">   90</span> </div>
<div class="line"><a id="l00091" name="l00091"></a><span class="lineno">   91</span>            __syncthreads();</div>
<div class="line"><a id="l00092" name="l00092"></a><span class="lineno">   92</span> </div>
<div class="line"><a id="l00093" name="l00093"></a><span class="lineno">   93</span>            <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i &lt; TILE_SIZE; i++)</div>
<div class="line"><a id="l00094" name="l00094"></a><span class="lineno">   94</span>                sum += As[threadIdx.y][i] * Bs[i][threadIdx.x];</div>
<div class="line"><a id="l00095" name="l00095"></a><span class="lineno">   95</span> </div>
<div class="line"><a id="l00096" name="l00096"></a><span class="lineno">   96</span>            __syncthreads();</div>
<div class="line"><a id="l00097" name="l00097"></a><span class="lineno">   97</span>        }</div>
<div class="line"><a id="l00098" name="l00098"></a><span class="lineno">   98</span> </div>
<div class="line"><a id="l00099" name="l00099"></a><span class="lineno">   99</span>        <span class="keywordflow">if</span> (row &lt; M &amp;&amp; col &lt; N)</div>
<div class="line"><a id="l00100" name="l00100"></a><span class="lineno">  100</span>            C[row * N + col + offset_c] = sum;</div>
<div class="line"><a id="l00101" name="l00101"></a><span class="lineno">  101</span>    }</div>
<div class="line"><a id="l00102" name="l00102"></a><span class="lineno">  102</span> </div>
<div class="foldopen" id="foldopen00103" data-start="{" data-end="}">
<div class="line"><a id="l00103" name="l00103"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#ae30a6e1de69588aa0c6eb8a5b8e6e826">  103</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#ae30a6e1de69588aa0c6eb8a5b8e6e826">GeneralMatrixMul</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B, <span class="keywordtype">float</span>* C,</div>
<div class="line"><a id="l00104" name="l00104"></a><span class="lineno">  104</span>                          <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> M,</div>
<div class="line"><a id="l00105" name="l00105"></a><span class="lineno">  105</span>                          <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> N,</div>
<div class="line"><a id="l00106" name="l00106"></a><span class="lineno">  106</span>                          <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> K,</div>
<div class="line"><a id="l00107" name="l00107"></a><span class="lineno">  107</span>                          <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_c,</div>
<div class="line"><a id="l00108" name="l00108"></a><span class="lineno">  108</span>                          <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_a,</div>
<div class="line"><a id="l00109" name="l00109"></a><span class="lineno">  109</span>                          <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_b) {</div>
<div class="line"><a id="l00110" name="l00110"></a><span class="lineno">  110</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(GeneralMatrixMulKernel, gridDim, blockDim, 0, C, A, B, M, N, K,</div>
<div class="line"><a id="l00111" name="l00111"></a><span class="lineno">  111</span>                                                offset_c, offset_a, offset_b);</div>
<div class="line"><a id="l00112" name="l00112"></a><span class="lineno">  112</span>    }</div>
</div>
<div class="line"><a id="l00113" name="l00113"></a><span class="lineno">  113</span> </div>
<div class="foldopen" id="foldopen00114" data-start="{" data-end="}">
<div class="line"><a id="l00114" name="l00114"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aa3720ebf4ae0cc9f4abbd1e32842191b">  114</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#ae30a6e1de69588aa0c6eb8a5b8e6e826">GeneralMatrixMul</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B, <span class="keywordtype">float</span>* C,</div>
<div class="line"><a id="l00115" name="l00115"></a><span class="lineno">  115</span>                          <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> M,</div>
<div class="line"><a id="l00116" name="l00116"></a><span class="lineno">  116</span>                          <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> N,</div>
<div class="line"><a id="l00117" name="l00117"></a><span class="lineno">  117</span>                          <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> K,</div>
<div class="line"><a id="l00118" name="l00118"></a><span class="lineno">  118</span>                          <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_c,</div>
<div class="line"><a id="l00119" name="l00119"></a><span class="lineno">  119</span>                          <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_a,</div>
<div class="line"><a id="l00120" name="l00120"></a><span class="lineno">  120</span>                          <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_b) {</div>
<div class="line"><a id="l00121" name="l00121"></a><span class="lineno">  121</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitParallel(GeneralMatrixMulKernel, gridDim, blockDim, 0, C, A, B,</div>
<div class="line"><a id="l00122" name="l00122"></a><span class="lineno">  122</span>                                                        offset_c, offset_a, offset_b, M, N, K);</div>
<div class="line"><a id="l00123" name="l00123"></a><span class="lineno">  123</span>    }</div>
</div>
<div class="line"><a id="l00124" name="l00124"></a><span class="lineno">  124</span> </div>
<div class="line"><a id="l00125" name="l00125"></a><span class="lineno">  125</span>    __global__ <span class="keywordtype">void</span> TransposeKernel(<span class="keywordtype">float</span>* d_B, <span class="keyword">const</span> <span class="keywordtype">float</span>* d_A,</div>
<div class="line"><a id="l00126" name="l00126"></a><span class="lineno">  126</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> rows,</div>
<div class="line"><a id="l00127" name="l00127"></a><span class="lineno">  127</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> cols,</div>
<div class="line"><a id="l00128" name="l00128"></a><span class="lineno">  128</span>                                    <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l00129" name="l00129"></a><span class="lineno">  129</span>        __shared__ <span class="keywordtype">float</span> tile[TILE_SIZE][TILE_SIZE];</div>
<div class="line"><a id="l00130" name="l00130"></a><span class="lineno">  130</span> </div>
<div class="line"><a id="l00131" name="l00131"></a><span class="lineno">  131</span>        <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> row = blockIdx.y * TILE_SIZE + threadIdx.y;</div>
<div class="line"><a id="l00132" name="l00132"></a><span class="lineno">  132</span>        <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> col = blockIdx.x * TILE_SIZE + threadIdx.x;</div>
<div class="line"><a id="l00133" name="l00133"></a><span class="lineno">  133</span> </div>
<div class="line"><a id="l00134" name="l00134"></a><span class="lineno">  134</span>        <span class="keywordflow">if</span> (row &lt; rows &amp;&amp; col &lt; cols)</div>
<div class="line"><a id="l00135" name="l00135"></a><span class="lineno">  135</span>            tile[threadIdx.y][threadIdx.x] = d_A[row * cols + col + offset];</div>
<div class="line"><a id="l00136" name="l00136"></a><span class="lineno">  136</span>        <span class="keywordflow">else</span></div>
<div class="line"><a id="l00137" name="l00137"></a><span class="lineno">  137</span>            tile[threadIdx.y][threadIdx.x] = 0.0f;</div>
<div class="line"><a id="l00138" name="l00138"></a><span class="lineno">  138</span> </div>
<div class="line"><a id="l00139" name="l00139"></a><span class="lineno">  139</span>        __syncthreads();</div>
<div class="line"><a id="l00140" name="l00140"></a><span class="lineno">  140</span> </div>
<div class="line"><a id="l00141" name="l00141"></a><span class="lineno">  141</span>        row = blockIdx.x * TILE_SIZE + threadIdx.y;</div>
<div class="line"><a id="l00142" name="l00142"></a><span class="lineno">  142</span>        col = blockIdx.y * TILE_SIZE + threadIdx.x;</div>
<div class="line"><a id="l00143" name="l00143"></a><span class="lineno">  143</span>        <span class="keywordflow">if</span> (row &lt; cols &amp;&amp; col &lt; rows)</div>
<div class="line"><a id="l00144" name="l00144"></a><span class="lineno">  144</span>            d_B[row * rows + col + offset] = tile[threadIdx.x][threadIdx.y];</div>
<div class="line"><a id="l00145" name="l00145"></a><span class="lineno">  145</span>    }</div>
<div class="line"><a id="l00146" name="l00146"></a><span class="lineno">  146</span> </div>
<div class="foldopen" id="foldopen00147" data-start="{" data-end="}">
<div class="line"><a id="l00147" name="l00147"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#afe3f38f788c735b7eb718443eb0fd094">  147</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#afe3f38f788c735b7eb718443eb0fd094">Transpose</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* d_A, <span class="keywordtype">float</span>* d_B,</div>
<div class="line"><a id="l00148" name="l00148"></a><span class="lineno">  148</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> rows,</div>
<div class="line"><a id="l00149" name="l00149"></a><span class="lineno">  149</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> cols,</div>
<div class="line"><a id="l00150" name="l00150"></a><span class="lineno">  150</span>                   <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l00151" name="l00151"></a><span class="lineno">  151</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(TransposeKernel, gridDim, blockDim, 0, d_B, d_A, rows, cols, offset);</div>
<div class="line"><a id="l00152" name="l00152"></a><span class="lineno">  152</span>    }</div>
</div>
<div class="line"><a id="l00153" name="l00153"></a><span class="lineno">  153</span> </div>
<div class="foldopen" id="foldopen00154" data-start="{" data-end="}">
<div class="line"><a id="l00154" name="l00154"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a16823e30ad99965b64a03e2d4a91a699">  154</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#afe3f38f788c735b7eb718443eb0fd094">Transpose</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* d_A, <span class="keywordtype">float</span>* d_B,</div>
<div class="line"><a id="l00155" name="l00155"></a><span class="lineno">  155</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> rows,</div>
<div class="line"><a id="l00156" name="l00156"></a><span class="lineno">  156</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> cols,</div>
<div class="line"><a id="l00157" name="l00157"></a><span class="lineno">  157</span>                   <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset) {</div>
<div class="line"><a id="l00158" name="l00158"></a><span class="lineno">  158</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitParallel(TransposeKernel, gridDim, blockDim, 0, d_B, d_A, offset, rows,</div>
<div class="line"><a id="l00159" name="l00159"></a><span class="lineno">  159</span>                                                        cols);</div>
<div class="line"><a id="l00160" name="l00160"></a><span class="lineno">  160</span>    }</div>
</div>
<div class="line"><a id="l00161" name="l00161"></a><span class="lineno">  161</span> </div>
<div class="line"><a id="l00162" name="l00162"></a><span class="lineno">  162</span>    __global__ <span class="keywordtype">void</span> ScalarMulKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">float</span> num,</div>
<div class="line"><a id="l00163" name="l00163"></a><span class="lineno">  163</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00164" name="l00164"></a><span class="lineno">  164</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00165" name="l00165"></a><span class="lineno">  165</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00166" name="l00166"></a><span class="lineno">  166</span>            out[idx] = in[idx] * num;</div>
<div class="line"><a id="l00167" name="l00167"></a><span class="lineno">  167</span>        }</div>
<div class="line"><a id="l00168" name="l00168"></a><span class="lineno">  168</span>    }</div>
<div class="line"><a id="l00169" name="l00169"></a><span class="lineno">  169</span> </div>
<div class="foldopen" id="foldopen00170" data-start="{" data-end="}">
<div class="line"><a id="l00170" name="l00170"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a5af716524e248c61f3dce227d8ef6e34">  170</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a5af716524e248c61f3dce227d8ef6e34">ScalarMul</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">float</span> num,</div>
<div class="line"><a id="l00171" name="l00171"></a><span class="lineno">  171</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00172" name="l00172"></a><span class="lineno">  172</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(ScalarMulKernel, gridDim, blockDim, 0, out, in, num, n);</div>
<div class="line"><a id="l00173" name="l00173"></a><span class="lineno">  173</span>    }</div>
</div>
<div class="line"><a id="l00174" name="l00174"></a><span class="lineno">  174</span> </div>
<div class="line"><a id="l00175" name="l00175"></a><span class="lineno">  175</span>    __global__ <span class="keywordtype">void</span> ScalarDivKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">float</span> num,</div>
<div class="line"><a id="l00176" name="l00176"></a><span class="lineno">  176</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00177" name="l00177"></a><span class="lineno">  177</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00178" name="l00178"></a><span class="lineno">  178</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00179" name="l00179"></a><span class="lineno">  179</span>            out[idx] = in[idx] / num;</div>
<div class="line"><a id="l00180" name="l00180"></a><span class="lineno">  180</span>        }</div>
<div class="line"><a id="l00181" name="l00181"></a><span class="lineno">  181</span>    }</div>
<div class="line"><a id="l00182" name="l00182"></a><span class="lineno">  182</span> </div>
<div class="foldopen" id="foldopen00183" data-start="{" data-end="}">
<div class="line"><a id="l00183" name="l00183"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a27bc4025be4253d5fffae2bf1b43b3af">  183</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a27bc4025be4253d5fffae2bf1b43b3af">ScalarDiv</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">float</span> num,</div>
<div class="line"><a id="l00184" name="l00184"></a><span class="lineno">  184</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00185" name="l00185"></a><span class="lineno">  185</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(ScalarDivKernel, gridDim, blockDim, 0, out, in, num, n);</div>
<div class="line"><a id="l00186" name="l00186"></a><span class="lineno">  186</span>    }</div>
</div>
<div class="line"><a id="l00187" name="l00187"></a><span class="lineno">  187</span> </div>
<div class="line"><a id="l00188" name="l00188"></a><span class="lineno">  188</span>    __global__ <span class="keywordtype">void</span> ScalarAddKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">float</span> num,</div>
<div class="line"><a id="l00189" name="l00189"></a><span class="lineno">  189</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00190" name="l00190"></a><span class="lineno">  190</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00191" name="l00191"></a><span class="lineno">  191</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00192" name="l00192"></a><span class="lineno">  192</span>            out[idx] = in[idx] + num;</div>
<div class="line"><a id="l00193" name="l00193"></a><span class="lineno">  193</span>        }</div>
<div class="line"><a id="l00194" name="l00194"></a><span class="lineno">  194</span>    }</div>
<div class="line"><a id="l00195" name="l00195"></a><span class="lineno">  195</span> </div>
<div class="foldopen" id="foldopen00196" data-start="{" data-end="}">
<div class="line"><a id="l00196" name="l00196"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a56f84e531825be8b2b0974c2488eb765">  196</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a56f84e531825be8b2b0974c2488eb765">ScalarAdd</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">float</span> num,</div>
<div class="line"><a id="l00197" name="l00197"></a><span class="lineno">  197</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00198" name="l00198"></a><span class="lineno">  198</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(ScalarAddKernel, gridDim, blockDim, 0, out, in, num, n);</div>
<div class="line"><a id="l00199" name="l00199"></a><span class="lineno">  199</span>    }</div>
</div>
<div class="line"><a id="l00200" name="l00200"></a><span class="lineno">  200</span> </div>
<div class="line"><a id="l00201" name="l00201"></a><span class="lineno">  201</span>    __global__ <span class="keywordtype">void</span> NegationKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00202" name="l00202"></a><span class="lineno">  202</span>                                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00203" name="l00203"></a><span class="lineno">  203</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00204" name="l00204"></a><span class="lineno">  204</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00205" name="l00205"></a><span class="lineno">  205</span>            out[idx] = -in[idx];</div>
<div class="line"><a id="l00206" name="l00206"></a><span class="lineno">  206</span>        }</div>
<div class="line"><a id="l00207" name="l00207"></a><span class="lineno">  207</span>    }</div>
<div class="line"><a id="l00208" name="l00208"></a><span class="lineno">  208</span> </div>
<div class="foldopen" id="foldopen00209" data-start="{" data-end="}">
<div class="line"><a id="l00209" name="l00209"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#af7069a420e81babb49b1bc009333d053">  209</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#af7069a420e81babb49b1bc009333d053">Negation</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00210" name="l00210"></a><span class="lineno">  210</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(NegationKernel, gridDim, blockDim, 0, out, in, n);</div>
<div class="line"><a id="l00211" name="l00211"></a><span class="lineno">  211</span>    }</div>
</div>
<div class="line"><a id="l00212" name="l00212"></a><span class="lineno">  212</span> </div>
<div class="line"><a id="l00213" name="l00213"></a><span class="lineno">  213</span>    __global__ <span class="keywordtype">void</span></div>
<div class="line"><a id="l00214" name="l00214"></a><span class="lineno">  214</span>    RecipKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00215" name="l00215"></a><span class="lineno">  215</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00216" name="l00216"></a><span class="lineno">  216</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00217" name="l00217"></a><span class="lineno">  217</span>            <span class="keywordflow">if</span> (in[idx] == 0) {</div>
<div class="line"><a id="l00218" name="l00218"></a><span class="lineno">  218</span>                out[idx] = 0.0f;</div>
<div class="line"><a id="l00219" name="l00219"></a><span class="lineno">  219</span>            }</div>
<div class="line"><a id="l00220" name="l00220"></a><span class="lineno">  220</span>            <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l00221" name="l00221"></a><span class="lineno">  221</span>                out[idx] = 1.0f / in[idx];</div>
<div class="line"><a id="l00222" name="l00222"></a><span class="lineno">  222</span>            }</div>
<div class="line"><a id="l00223" name="l00223"></a><span class="lineno">  223</span>        }</div>
<div class="line"><a id="l00224" name="l00224"></a><span class="lineno">  224</span>    }</div>
<div class="line"><a id="l00225" name="l00225"></a><span class="lineno">  225</span> </div>
<div class="foldopen" id="foldopen00226" data-start="{" data-end="}">
<div class="line"><a id="l00226" name="l00226"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#adc047e65307dbc711235f637227b7d10">  226</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#adc047e65307dbc711235f637227b7d10">Recip</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00227" name="l00227"></a><span class="lineno">  227</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(RecipKernel, gridDim, blockDim, 0, out, in, n);</div>
<div class="line"><a id="l00228" name="l00228"></a><span class="lineno">  228</span>    }</div>
</div>
<div class="line"><a id="l00229" name="l00229"></a><span class="lineno">  229</span> </div>
<div class="line"><a id="l00230" name="l00230"></a><span class="lineno">  230</span>    __global__ <span class="keywordtype">void</span> RectifiedLinearUnitKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00231" name="l00231"></a><span class="lineno">  231</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00232" name="l00232"></a><span class="lineno">  232</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00233" name="l00233"></a><span class="lineno">  233</span>            out[idx] = in[idx] &gt; 0 ? in[idx] : 0;</div>
<div class="line"><a id="l00234" name="l00234"></a><span class="lineno">  234</span>        }</div>
<div class="line"><a id="l00235" name="l00235"></a><span class="lineno">  235</span>    }</div>
<div class="line"><a id="l00236" name="l00236"></a><span class="lineno">  236</span> </div>
<div class="foldopen" id="foldopen00237" data-start="{" data-end="}">
<div class="line"><a id="l00237" name="l00237"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a8855f411733f7de29d013f4ad40096c9">  237</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a8855f411733f7de29d013f4ad40096c9">RectifiedLinearUnit</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00238" name="l00238"></a><span class="lineno">  238</span>                             <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00239" name="l00239"></a><span class="lineno">  239</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(RectifiedLinearUnitKernel, gridDim, blockDim, 0, out, in, n);</div>
<div class="line"><a id="l00240" name="l00240"></a><span class="lineno">  240</span>    }</div>
</div>
<div class="line"><a id="l00241" name="l00241"></a><span class="lineno">  241</span> </div>
<div class="line"><a id="l00242" name="l00242"></a><span class="lineno">  242</span>    __global__ <span class="keywordtype">void</span> ReLUBackwardKernel(<span class="keywordtype">float</span>* A_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* A,</div>
<div class="line"><a id="l00243" name="l00243"></a><span class="lineno">  243</span>                                       <span class="keyword">const</span> <span class="keywordtype">float</span>* B_grad, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00244" name="l00244"></a><span class="lineno">  244</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00245" name="l00245"></a><span class="lineno">  245</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00246" name="l00246"></a><span class="lineno">  246</span>            A_grad[idx] = A[idx] &gt; 0 ? B_grad[idx] : 0;</div>
<div class="line"><a id="l00247" name="l00247"></a><span class="lineno">  247</span>        }</div>
<div class="line"><a id="l00248" name="l00248"></a><span class="lineno">  248</span>    }</div>
<div class="line"><a id="l00249" name="l00249"></a><span class="lineno">  249</span> </div>
<div class="foldopen" id="foldopen00250" data-start="{" data-end="}">
<div class="line"><a id="l00250" name="l00250"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a4ddfc808de99fe831e74a3bd3f9bbdaf">  250</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a4ddfc808de99fe831e74a3bd3f9bbdaf">ReLUBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A_grad, <span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00251" name="l00251"></a><span class="lineno">  251</span>                      <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00252" name="l00252"></a><span class="lineno">  252</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(ReLUBackwardKernel, gridDim, blockDim, 0, A_grad, A, B_grad, n);</div>
<div class="line"><a id="l00253" name="l00253"></a><span class="lineno">  253</span>    }</div>
</div>
<div class="line"><a id="l00254" name="l00254"></a><span class="lineno">  254</span> </div>
<div class="line"><a id="l00255" name="l00255"></a><span class="lineno">  255</span>    __global__ <span class="keywordtype">void</span> SigmoidKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00256" name="l00256"></a><span class="lineno">  256</span>                                  <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00257" name="l00257"></a><span class="lineno">  257</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00258" name="l00258"></a><span class="lineno">  258</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00259" name="l00259"></a><span class="lineno">  259</span>            out[idx] = 1.0f / (1.0f + __expf(-in[idx]));</div>
<div class="line"><a id="l00260" name="l00260"></a><span class="lineno">  260</span>        }</div>
<div class="line"><a id="l00261" name="l00261"></a><span class="lineno">  261</span>    }</div>
<div class="line"><a id="l00262" name="l00262"></a><span class="lineno">  262</span> </div>
<div class="foldopen" id="foldopen00263" data-start="{" data-end="}">
<div class="line"><a id="l00263" name="l00263"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a21bbbcf6d97bfaccc828ce7736814bd4">  263</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a21bbbcf6d97bfaccc828ce7736814bd4">Sigmoid</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00264" name="l00264"></a><span class="lineno">  264</span>                 <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00265" name="l00265"></a><span class="lineno">  265</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(SigmoidKernel, gridDim, blockDim, 0, out, in, n);</div>
<div class="line"><a id="l00266" name="l00266"></a><span class="lineno">  266</span>    }</div>
</div>
<div class="line"><a id="l00267" name="l00267"></a><span class="lineno">  267</span> </div>
<div class="line"><a id="l00268" name="l00268"></a><span class="lineno">  268</span>    __global__ <span class="keywordtype">void</span> SigmoidBackwardKernel(<span class="keywordtype">float</span>* A_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* B,</div>
<div class="line"><a id="l00269" name="l00269"></a><span class="lineno">  269</span>                                          <span class="keyword">const</span> <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00270" name="l00270"></a><span class="lineno">  270</span>                                          <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00271" name="l00271"></a><span class="lineno">  271</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00272" name="l00272"></a><span class="lineno">  272</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00273" name="l00273"></a><span class="lineno">  273</span>            A_grad[idx] = B[idx] * (1.0f - B[idx]) * B_grad[idx];</div>
<div class="line"><a id="l00274" name="l00274"></a><span class="lineno">  274</span>        }</div>
<div class="line"><a id="l00275" name="l00275"></a><span class="lineno">  275</span>    }</div>
<div class="line"><a id="l00276" name="l00276"></a><span class="lineno">  276</span> </div>
<div class="foldopen" id="foldopen00277" data-start="{" data-end="}">
<div class="line"><a id="l00277" name="l00277"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aff1f9f1bf9fb677024bd2b565fab9801">  277</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aff1f9f1bf9fb677024bd2b565fab9801">SigmoidBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A_grad, <span class="keywordtype">float</span>* B, <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00278" name="l00278"></a><span class="lineno">  278</span>                         <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00279" name="l00279"></a><span class="lineno">  279</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(SigmoidBackwardKernel, gridDim, blockDim, 0, A_grad, B, B_grad, n);</div>
<div class="line"><a id="l00280" name="l00280"></a><span class="lineno">  280</span>    }</div>
</div>
<div class="line"><a id="l00281" name="l00281"></a><span class="lineno">  281</span> </div>
<div class="line"><a id="l00282" name="l00282"></a><span class="lineno">  282</span>    __global__ <span class="keywordtype">void</span> TanhKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00283" name="l00283"></a><span class="lineno">  283</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00284" name="l00284"></a><span class="lineno">  284</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00285" name="l00285"></a><span class="lineno">  285</span>            out[idx] = (__expf(in[idx]) - __expf(-in[idx])) / (__expf(in[idx]) + __expf(-in[idx]));</div>
<div class="line"><a id="l00286" name="l00286"></a><span class="lineno">  286</span>        }</div>
<div class="line"><a id="l00287" name="l00287"></a><span class="lineno">  287</span>    }</div>
<div class="line"><a id="l00288" name="l00288"></a><span class="lineno">  288</span> </div>
<div class="foldopen" id="foldopen00289" data-start="{" data-end="}">
<div class="line"><a id="l00289" name="l00289"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aeb7d10939b25508e0b5db1fe44f4b467">  289</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aeb7d10939b25508e0b5db1fe44f4b467">Tanh</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00290" name="l00290"></a><span class="lineno">  290</span>              <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00291" name="l00291"></a><span class="lineno">  291</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(TanhKernel, gridDim, blockDim, 0, out, in, n);</div>
<div class="line"><a id="l00292" name="l00292"></a><span class="lineno">  292</span>    }</div>
</div>
<div class="line"><a id="l00293" name="l00293"></a><span class="lineno">  293</span> </div>
<div class="line"><a id="l00294" name="l00294"></a><span class="lineno">  294</span>    __global__ <span class="keywordtype">void</span> TanhBackwardKernel(<span class="keywordtype">float</span>* A_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* B,</div>
<div class="line"><a id="l00295" name="l00295"></a><span class="lineno">  295</span>                                       <span class="keyword">const</span> <span class="keywordtype">float</span>* B_grad, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00296" name="l00296"></a><span class="lineno">  296</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00297" name="l00297"></a><span class="lineno">  297</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00298" name="l00298"></a><span class="lineno">  298</span>            A_grad[idx] = (1.0f - B[idx] * B[idx]) * B_grad[idx];</div>
<div class="line"><a id="l00299" name="l00299"></a><span class="lineno">  299</span>        }</div>
<div class="line"><a id="l00300" name="l00300"></a><span class="lineno">  300</span>    }</div>
<div class="line"><a id="l00301" name="l00301"></a><span class="lineno">  301</span> </div>
<div class="foldopen" id="foldopen00302" data-start="{" data-end="}">
<div class="line"><a id="l00302" name="l00302"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a90d501e72361b7341f36394af0f27c74">  302</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a90d501e72361b7341f36394af0f27c74">TanhBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A_grad, <span class="keywordtype">float</span>* B, <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00303" name="l00303"></a><span class="lineno">  303</span>                      <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00304" name="l00304"></a><span class="lineno">  304</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(TanhBackwardKernel, gridDim, blockDim, 0, A_grad, B, B_grad, n);</div>
<div class="line"><a id="l00305" name="l00305"></a><span class="lineno">  305</span>    }</div>
</div>
<div class="line"><a id="l00306" name="l00306"></a><span class="lineno">  306</span> </div>
<div class="line"><a id="l00307" name="l00307"></a><span class="lineno">  307</span>    __global__ <span class="keywordtype">void</span> LeakyReLUKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00308" name="l00308"></a><span class="lineno">  308</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha) {</div>
<div class="line"><a id="l00309" name="l00309"></a><span class="lineno">  309</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00310" name="l00310"></a><span class="lineno">  310</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00311" name="l00311"></a><span class="lineno">  311</span>            out[idx] = in[idx] &gt; 0 ? in[idx] : alpha * in[idx];</div>
<div class="line"><a id="l00312" name="l00312"></a><span class="lineno">  312</span>        }</div>
<div class="line"><a id="l00313" name="l00313"></a><span class="lineno">  313</span>    }</div>
<div class="line"><a id="l00314" name="l00314"></a><span class="lineno">  314</span> </div>
<div class="foldopen" id="foldopen00315" data-start="{" data-end="}">
<div class="line"><a id="l00315" name="l00315"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a04246c5218530f789a0ed4811b7ef3f3">  315</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a04246c5218530f789a0ed4811b7ef3f3">LeakyReLU</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00316" name="l00316"></a><span class="lineno">  316</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha) {</div>
<div class="line"><a id="l00317" name="l00317"></a><span class="lineno">  317</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(LeakyReLUKernel, gridDim, blockDim, 0, out, in, n, alpha);</div>
<div class="line"><a id="l00318" name="l00318"></a><span class="lineno">  318</span>    }</div>
</div>
<div class="line"><a id="l00319" name="l00319"></a><span class="lineno">  319</span> </div>
<div class="line"><a id="l00320" name="l00320"></a><span class="lineno">  320</span>    __global__ <span class="keywordtype">void</span> LeakyReLUBackwardKernel(<span class="keywordtype">float</span>* A_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* A,</div>
<div class="line"><a id="l00321" name="l00321"></a><span class="lineno">  321</span>                                            <span class="keyword">const</span> <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00322" name="l00322"></a><span class="lineno">  322</span>                                            <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l00323" name="l00323"></a><span class="lineno">  323</span>                                            <span class="keyword">const</span> <span class="keywordtype">float</span> alpha) {</div>
<div class="line"><a id="l00324" name="l00324"></a><span class="lineno">  324</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00325" name="l00325"></a><span class="lineno">  325</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00326" name="l00326"></a><span class="lineno">  326</span>            A_grad[idx] = A[idx] &gt; 0 ? B_grad[idx] : alpha * B_grad[idx];</div>
<div class="line"><a id="l00327" name="l00327"></a><span class="lineno">  327</span>        }</div>
<div class="line"><a id="l00328" name="l00328"></a><span class="lineno">  328</span>    }</div>
<div class="line"><a id="l00329" name="l00329"></a><span class="lineno">  329</span> </div>
<div class="foldopen" id="foldopen00330" data-start="{" data-end="}">
<div class="line"><a id="l00330" name="l00330"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a7eade95ddcf48141d69bb19803b22d51">  330</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a7eade95ddcf48141d69bb19803b22d51">LeakyReLUBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A_grad, <span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00331" name="l00331"></a><span class="lineno">  331</span>                           <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha) {</div>
<div class="line"><a id="l00332" name="l00332"></a><span class="lineno">  332</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(LeakyReLUBackwardKernel, gridDim, blockDim, 0, A_grad, A, B_grad, n,</div>
<div class="line"><a id="l00333" name="l00333"></a><span class="lineno">  333</span>                                                alpha);</div>
<div class="line"><a id="l00334" name="l00334"></a><span class="lineno">  334</span>    }</div>
</div>
<div class="line"><a id="l00335" name="l00335"></a><span class="lineno">  335</span> </div>
<div class="line"><a id="l00336" name="l00336"></a><span class="lineno">  336</span>    __global__ <span class="keywordtype">void</span></div>
<div class="line"><a id="l00337" name="l00337"></a><span class="lineno">  337</span>    SwishKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00338" name="l00338"></a><span class="lineno">  338</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00339" name="l00339"></a><span class="lineno">  339</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00340" name="l00340"></a><span class="lineno">  340</span>            out[idx] = in[idx] / (1.0f + __expf(-in[idx]));</div>
<div class="line"><a id="l00341" name="l00341"></a><span class="lineno">  341</span>        }</div>
<div class="line"><a id="l00342" name="l00342"></a><span class="lineno">  342</span>    }</div>
<div class="line"><a id="l00343" name="l00343"></a><span class="lineno">  343</span> </div>
<div class="foldopen" id="foldopen00344" data-start="{" data-end="}">
<div class="line"><a id="l00344" name="l00344"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a997aa5460fd64fadf9b701fbf73e3fb2">  344</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a997aa5460fd64fadf9b701fbf73e3fb2">Swish</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00345" name="l00345"></a><span class="lineno">  345</span>               <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00346" name="l00346"></a><span class="lineno">  346</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(SwishKernel, gridDim, blockDim, 0, out, in, n);</div>
<div class="line"><a id="l00347" name="l00347"></a><span class="lineno">  347</span>    }</div>
</div>
<div class="line"><a id="l00348" name="l00348"></a><span class="lineno">  348</span> </div>
<div class="line"><a id="l00349" name="l00349"></a><span class="lineno">  349</span>    __global__ <span class="keywordtype">void</span> SwishBackwardKernel(<span class="keywordtype">float</span>* A_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* A,</div>
<div class="line"><a id="l00350" name="l00350"></a><span class="lineno">  350</span>                                        <span class="keyword">const</span> <span class="keywordtype">float</span>* B, <span class="keyword">const</span> <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00351" name="l00351"></a><span class="lineno">  351</span>                                        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00352" name="l00352"></a><span class="lineno">  352</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00353" name="l00353"></a><span class="lineno">  353</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00354" name="l00354"></a><span class="lineno">  354</span>            A_grad[idx] = 1.0f / (1.0f + __expf(-A[idx])) + B[idx] * (1.0f - B[idx]) *</div>
<div class="line"><a id="l00355" name="l00355"></a><span class="lineno">  355</span>                B_grad[idx];</div>
<div class="line"><a id="l00356" name="l00356"></a><span class="lineno">  356</span>        }</div>
<div class="line"><a id="l00357" name="l00357"></a><span class="lineno">  357</span>    }</div>
<div class="line"><a id="l00358" name="l00358"></a><span class="lineno">  358</span> </div>
<div class="foldopen" id="foldopen00359" data-start="{" data-end="}">
<div class="line"><a id="l00359" name="l00359"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a6c5a4b54442aab42df5afe8688e71596">  359</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a6c5a4b54442aab42df5afe8688e71596">SwishBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A_grad, <span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B,</div>
<div class="line"><a id="l00360" name="l00360"></a><span class="lineno">  360</span>                       <span class="keywordtype">float</span>* B_grad, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00361" name="l00361"></a><span class="lineno">  361</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(SwishBackwardKernel, gridDim, blockDim, 0, A_grad, A, B, B_grad, n);</div>
<div class="line"><a id="l00362" name="l00362"></a><span class="lineno">  362</span>    }</div>
</div>
<div class="line"><a id="l00363" name="l00363"></a><span class="lineno">  363</span> </div>
<div class="line"><a id="l00364" name="l00364"></a><span class="lineno">  364</span>    __global__ <span class="keywordtype">void</span> ExponentialLinearUnitKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l00365" name="l00365"></a><span class="lineno">  365</span>                                                <span class="keyword">const</span> <span class="keywordtype">float</span> alpha) {</div>
<div class="line"><a id="l00366" name="l00366"></a><span class="lineno">  366</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00367" name="l00367"></a><span class="lineno">  367</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00368" name="l00368"></a><span class="lineno">  368</span>            out[idx] = in[idx] &gt; 0 ? in[idx] : alpha * (__expf(in[idx]) - 1);</div>
<div class="line"><a id="l00369" name="l00369"></a><span class="lineno">  369</span>        }</div>
<div class="line"><a id="l00370" name="l00370"></a><span class="lineno">  370</span>    }</div>
<div class="line"><a id="l00371" name="l00371"></a><span class="lineno">  371</span> </div>
<div class="foldopen" id="foldopen00372" data-start="{" data-end="}">
<div class="line"><a id="l00372" name="l00372"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a0e82aca250b46ac8ded8cae8936d7e38">  372</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a0e82aca250b46ac8ded8cae8936d7e38">ExponentialLinearUnit</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00373" name="l00373"></a><span class="lineno">  373</span>                               <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha) {</div>
<div class="line"><a id="l00374" name="l00374"></a><span class="lineno">  374</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(ExponentialLinearUnitKernel, gridDim, blockDim, 0, out, in, n, alpha);</div>
<div class="line"><a id="l00375" name="l00375"></a><span class="lineno">  375</span>    }</div>
</div>
<div class="line"><a id="l00376" name="l00376"></a><span class="lineno">  376</span> </div>
<div class="line"><a id="l00377" name="l00377"></a><span class="lineno">  377</span>    __global__ <span class="keywordtype">void</span> ELUBackwardKernel(<span class="keywordtype">float</span>* A_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* A,</div>
<div class="line"><a id="l00378" name="l00378"></a><span class="lineno">  378</span>                                      <span class="keyword">const</span> <span class="keywordtype">float</span>* B_grad, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l00379" name="l00379"></a><span class="lineno">  379</span>                                      <span class="keyword">const</span> <span class="keywordtype">float</span> alpha) {</div>
<div class="line"><a id="l00380" name="l00380"></a><span class="lineno">  380</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00381" name="l00381"></a><span class="lineno">  381</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00382" name="l00382"></a><span class="lineno">  382</span>            A_grad[idx] = A[idx] &gt; 0</div>
<div class="line"><a id="l00383" name="l00383"></a><span class="lineno">  383</span>                              ? B_grad[idx]</div>
<div class="line"><a id="l00384" name="l00384"></a><span class="lineno">  384</span>                              : alpha * __expf(A[idx]) * B_grad[idx];</div>
<div class="line"><a id="l00385" name="l00385"></a><span class="lineno">  385</span>        }</div>
<div class="line"><a id="l00386" name="l00386"></a><span class="lineno">  386</span>    }</div>
<div class="line"><a id="l00387" name="l00387"></a><span class="lineno">  387</span> </div>
<div class="foldopen" id="foldopen00388" data-start="{" data-end="}">
<div class="line"><a id="l00388" name="l00388"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aee8ca471aa260bd1fca5b1797e229f9f">  388</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aee8ca471aa260bd1fca5b1797e229f9f">ELUBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A_grad, <span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00389" name="l00389"></a><span class="lineno">  389</span>                     <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha) {</div>
<div class="line"><a id="l00390" name="l00390"></a><span class="lineno">  390</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(ELUBackwardKernel, gridDim, blockDim, 0, A_grad, A, B_grad, n, alpha);</div>
<div class="line"><a id="l00391" name="l00391"></a><span class="lineno">  391</span>    }</div>
</div>
<div class="line"><a id="l00392" name="l00392"></a><span class="lineno">  392</span> </div>
<div class="line"><a id="l00393" name="l00393"></a><span class="lineno">  393</span>    __global__ <span class="keywordtype">void</span> HardSigmoidKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00394" name="l00394"></a><span class="lineno">  394</span>                                      <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha,</div>
<div class="line"><a id="l00395" name="l00395"></a><span class="lineno">  395</span>                                      <span class="keyword">const</span> <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00396" name="l00396"></a><span class="lineno">  396</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00397" name="l00397"></a><span class="lineno">  397</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00398" name="l00398"></a><span class="lineno">  398</span>            out[idx] = in[idx] * alpha + beta;</div>
<div class="line"><a id="l00399" name="l00399"></a><span class="lineno">  399</span>            out[idx] = out[idx] &gt; 1.0f ? 1.0f : (out[idx] &lt; 0.0f ? 0.0f : out[idx]);</div>
<div class="line"><a id="l00400" name="l00400"></a><span class="lineno">  400</span>        }</div>
<div class="line"><a id="l00401" name="l00401"></a><span class="lineno">  401</span>    }</div>
<div class="line"><a id="l00402" name="l00402"></a><span class="lineno">  402</span> </div>
<div class="foldopen" id="foldopen00403" data-start="{" data-end="}">
<div class="line"><a id="l00403" name="l00403"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a52e449285e560185378234aecaf2f87c">  403</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a52e449285e560185378234aecaf2f87c">HardSigmoid</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00404" name="l00404"></a><span class="lineno">  404</span>                     <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha, <span class="keyword">const</span> <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00405" name="l00405"></a><span class="lineno">  405</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(HardSigmoidKernel, gridDim, blockDim, 0, out, in, n, alpha, beta);</div>
<div class="line"><a id="l00406" name="l00406"></a><span class="lineno">  406</span>    }</div>
</div>
<div class="line"><a id="l00407" name="l00407"></a><span class="lineno">  407</span> </div>
<div class="line"><a id="l00408" name="l00408"></a><span class="lineno">  408</span>    __global__ <span class="keywordtype">void</span> HardSigmoidBackwardKernel(<span class="keywordtype">float</span>* A_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* A,</div>
<div class="line"><a id="l00409" name="l00409"></a><span class="lineno">  409</span>                                              <span class="keyword">const</span> <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00410" name="l00410"></a><span class="lineno">  410</span>                                              <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l00411" name="l00411"></a><span class="lineno">  411</span>                                              <span class="keyword">const</span> <span class="keywordtype">float</span> alpha, <span class="keyword">const</span> <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00412" name="l00412"></a><span class="lineno">  412</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00413" name="l00413"></a><span class="lineno">  413</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00414" name="l00414"></a><span class="lineno">  414</span>            <span class="keywordtype">float</span> x = A[idx] * alpha + beta;</div>
<div class="line"><a id="l00415" name="l00415"></a><span class="lineno">  415</span>            <span class="keywordflow">if</span> (x &gt; 0.0f &amp;&amp; x &lt; 1.0f) {</div>
<div class="line"><a id="l00416" name="l00416"></a><span class="lineno">  416</span>                A_grad[idx] = B_grad[idx] * alpha;</div>
<div class="line"><a id="l00417" name="l00417"></a><span class="lineno">  417</span>            }</div>
<div class="line"><a id="l00418" name="l00418"></a><span class="lineno">  418</span>            <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l00419" name="l00419"></a><span class="lineno">  419</span>                A_grad[idx] = 0.0f;</div>
<div class="line"><a id="l00420" name="l00420"></a><span class="lineno">  420</span>            }</div>
<div class="line"><a id="l00421" name="l00421"></a><span class="lineno">  421</span>        }</div>
<div class="line"><a id="l00422" name="l00422"></a><span class="lineno">  422</span>    }</div>
<div class="line"><a id="l00423" name="l00423"></a><span class="lineno">  423</span> </div>
<div class="foldopen" id="foldopen00424" data-start="{" data-end="}">
<div class="line"><a id="l00424" name="l00424"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a43232f9472ad3b974351e59386208efa">  424</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a43232f9472ad3b974351e59386208efa">HardSigmoidBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A_grad, <span class="keywordtype">float</span>* A,</div>
<div class="line"><a id="l00425" name="l00425"></a><span class="lineno">  425</span>                             <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00426" name="l00426"></a><span class="lineno">  426</span>                             <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha, <span class="keyword">const</span> <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00427" name="l00427"></a><span class="lineno">  427</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(HardSigmoidBackwardKernel, gridDim, blockDim, 0, A_grad, A, B_grad, n,</div>
<div class="line"><a id="l00428" name="l00428"></a><span class="lineno">  428</span>                                                alpha, beta);</div>
<div class="line"><a id="l00429" name="l00429"></a><span class="lineno">  429</span>    }</div>
</div>
<div class="line"><a id="l00430" name="l00430"></a><span class="lineno">  430</span> </div>
<div class="line"><a id="l00431" name="l00431"></a><span class="lineno">  431</span>    __inline__ __device__ <span class="keywordtype">float</span> LiteHardSigmoid(<span class="keywordtype">float</span> x, <span class="keywordtype">float</span> alpha, <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00432" name="l00432"></a><span class="lineno">  432</span>        <span class="keywordtype">float</span> a = x * alpha + beta;</div>
<div class="line"><a id="l00433" name="l00433"></a><span class="lineno">  433</span>        <span class="keywordflow">return</span> a &gt; 1.0f ? 1.0f : (a &lt; 0.0f ? 0.0f : a);</div>
<div class="line"><a id="l00434" name="l00434"></a><span class="lineno">  434</span>    }</div>
<div class="line"><a id="l00435" name="l00435"></a><span class="lineno">  435</span> </div>
<div class="line"><a id="l00436" name="l00436"></a><span class="lineno">  436</span>    __global__ <span class="keywordtype">void</span> HardSwishKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00437" name="l00437"></a><span class="lineno">  437</span>                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha,</div>
<div class="line"><a id="l00438" name="l00438"></a><span class="lineno">  438</span>                                    <span class="keyword">const</span> <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00439" name="l00439"></a><span class="lineno">  439</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00440" name="l00440"></a><span class="lineno">  440</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00441" name="l00441"></a><span class="lineno">  441</span>            out[idx] = in[idx] * LiteHardSigmoid(in[idx], alpha, beta);</div>
<div class="line"><a id="l00442" name="l00442"></a><span class="lineno">  442</span>        }</div>
<div class="line"><a id="l00443" name="l00443"></a><span class="lineno">  443</span>    }</div>
<div class="line"><a id="l00444" name="l00444"></a><span class="lineno">  444</span> </div>
<div class="foldopen" id="foldopen00445" data-start="{" data-end="}">
<div class="line"><a id="l00445" name="l00445"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aef9c028ed356b5684e103639bb23bcf0">  445</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aef9c028ed356b5684e103639bb23bcf0">HardSwish</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00446" name="l00446"></a><span class="lineno">  446</span>                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha, <span class="keyword">const</span> <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00447" name="l00447"></a><span class="lineno">  447</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(HardSwishKernel, gridDim, blockDim, 0, out, in, n, alpha, beta);</div>
<div class="line"><a id="l00448" name="l00448"></a><span class="lineno">  448</span>    }</div>
</div>
<div class="line"><a id="l00449" name="l00449"></a><span class="lineno">  449</span> </div>
<div class="line"><a id="l00450" name="l00450"></a><span class="lineno">  450</span>    __global__ <span class="keywordtype">void</span> HardSwishBackwardKernel(<span class="keywordtype">float</span>* A_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* A,</div>
<div class="line"><a id="l00451" name="l00451"></a><span class="lineno">  451</span>                                            <span class="keyword">const</span> <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00452" name="l00452"></a><span class="lineno">  452</span>                                            <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l00453" name="l00453"></a><span class="lineno">  453</span>                                            <span class="keyword">const</span> <span class="keywordtype">float</span> alpha, <span class="keyword">const</span> <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00454" name="l00454"></a><span class="lineno">  454</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00455" name="l00455"></a><span class="lineno">  455</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00456" name="l00456"></a><span class="lineno">  456</span>            A_grad[idx] = LiteHardSigmoid(A[idx], alpha, beta) + B_grad[idx] * A[idx] *</div>
<div class="line"><a id="l00457" name="l00457"></a><span class="lineno">  457</span>                alpha * (1 - LiteHardSigmoid(</div>
<div class="line"><a id="l00458" name="l00458"></a><span class="lineno">  458</span>                    A[idx], alpha, beta));</div>
<div class="line"><a id="l00459" name="l00459"></a><span class="lineno">  459</span>        }</div>
<div class="line"><a id="l00460" name="l00460"></a><span class="lineno">  460</span>    }</div>
<div class="line"><a id="l00461" name="l00461"></a><span class="lineno">  461</span> </div>
<div class="foldopen" id="foldopen00462" data-start="{" data-end="}">
<div class="line"><a id="l00462" name="l00462"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a455365870d43ff26687a731d15c4cdff">  462</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a455365870d43ff26687a731d15c4cdff">HardSwishBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* A_grad, <span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B_grad,</div>
<div class="line"><a id="l00463" name="l00463"></a><span class="lineno">  463</span>                           <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">float</span> alpha, <span class="keyword">const</span> <span class="keywordtype">float</span> beta) {</div>
<div class="line"><a id="l00464" name="l00464"></a><span class="lineno">  464</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(HardSwishBackwardKernel, gridDim, blockDim, 0, A_grad, A, B_grad, n,</div>
<div class="line"><a id="l00465" name="l00465"></a><span class="lineno">  465</span>                                                alpha, beta);</div>
<div class="line"><a id="l00466" name="l00466"></a><span class="lineno">  466</span>    }</div>
</div>
<div class="line"><a id="l00467" name="l00467"></a><span class="lineno">  467</span> </div>
<div class="line"><a id="l00468" name="l00468"></a><span class="lineno">  468</span>    __inline__ __device__ <span class="keywordtype">float</span> warpReduce(<span class="keywordtype">float</span> localSum) {</div>
<div class="line"><a id="l00469" name="l00469"></a><span class="lineno">  469</span>        localSum += __shfl_xor_sync(FULL_MASK, localSum, 16);</div>
<div class="line"><a id="l00470" name="l00470"></a><span class="lineno">  470</span>        localSum += __shfl_xor_sync(FULL_MASK, localSum, 8);</div>
<div class="line"><a id="l00471" name="l00471"></a><span class="lineno">  471</span>        localSum += __shfl_xor_sync(FULL_MASK, localSum, 4);</div>
<div class="line"><a id="l00472" name="l00472"></a><span class="lineno">  472</span>        localSum += __shfl_xor_sync(FULL_MASK, localSum, 2);</div>
<div class="line"><a id="l00473" name="l00473"></a><span class="lineno">  473</span>        localSum += __shfl_xor_sync(FULL_MASK, localSum, 1);</div>
<div class="line"><a id="l00474" name="l00474"></a><span class="lineno">  474</span>        <span class="keywordflow">return</span> localSum;</div>
<div class="line"><a id="l00475" name="l00475"></a><span class="lineno">  475</span>    }</div>
<div class="line"><a id="l00476" name="l00476"></a><span class="lineno">  476</span> </div>
<div class="line"><a id="l00477" name="l00477"></a><span class="lineno">  477</span>    __global__ <span class="keywordtype">void</span> SummationExpKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* g_data,</div>
<div class="line"><a id="l00478" name="l00478"></a><span class="lineno">  478</span>                                       <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l00479" name="l00479"></a><span class="lineno">  479</span>        <span class="keyword">extern</span> __shared__ <span class="keywordtype">float</span> sdata[];</div>
<div class="line"><a id="l00480" name="l00480"></a><span class="lineno">  480</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00481" name="l00481"></a><span class="lineno">  481</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> tid = threadIdx.x;</div>
<div class="line"><a id="l00482" name="l00482"></a><span class="lineno">  482</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> warpIdx = tid / WARP_SIZE;</div>
<div class="line"><a id="l00483" name="l00483"></a><span class="lineno">  483</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> laneIdx = tid % WARP_SIZE;</div>
<div class="line"><a id="l00484" name="l00484"></a><span class="lineno">  484</span>        <span class="keywordtype">float</span> localSum = 0.0f;</div>
<div class="line"><a id="l00485" name="l00485"></a><span class="lineno">  485</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00486" name="l00486"></a><span class="lineno">  486</span>            localSum = __expf(g_data[idx + offset]);</div>
<div class="line"><a id="l00487" name="l00487"></a><span class="lineno">  487</span>        }</div>
<div class="line"><a id="l00488" name="l00488"></a><span class="lineno">  488</span>        <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l00489" name="l00489"></a><span class="lineno">  489</span>            localSum = 0.0f;</div>
<div class="line"><a id="l00490" name="l00490"></a><span class="lineno">  490</span>        }</div>
<div class="line"><a id="l00491" name="l00491"></a><span class="lineno">  491</span>        __syncthreads();</div>
<div class="line"><a id="l00492" name="l00492"></a><span class="lineno">  492</span>        <span class="comment">// Warp Reduce</span></div>
<div class="line"><a id="l00493" name="l00493"></a><span class="lineno">  493</span>        localSum = warpReduce(localSum);</div>
<div class="line"><a id="l00494" name="l00494"></a><span class="lineno">  494</span>        <span class="keywordflow">if</span> (laneIdx == 0) {</div>
<div class="line"><a id="l00495" name="l00495"></a><span class="lineno">  495</span>            sdata[warpIdx] = localSum;</div>
<div class="line"><a id="l00496" name="l00496"></a><span class="lineno">  496</span>        }</div>
<div class="line"><a id="l00497" name="l00497"></a><span class="lineno">  497</span>        __syncthreads();</div>
<div class="line"><a id="l00498" name="l00498"></a><span class="lineno">  498</span>        localSum = (tid &lt; blockDim.x / WARP_SIZE) ? sdata[laneIdx] : 0.0f;</div>
<div class="line"><a id="l00499" name="l00499"></a><span class="lineno">  499</span>        <span class="comment">// Block Reduce</span></div>
<div class="line"><a id="l00500" name="l00500"></a><span class="lineno">  500</span>        <span class="keywordflow">if</span> (warpIdx == 0) {</div>
<div class="line"><a id="l00501" name="l00501"></a><span class="lineno">  501</span>            localSum = warpReduce(localSum);</div>
<div class="line"><a id="l00502" name="l00502"></a><span class="lineno">  502</span>        }</div>
<div class="line"><a id="l00503" name="l00503"></a><span class="lineno">  503</span>        __syncthreads();</div>
<div class="line"><a id="l00504" name="l00504"></a><span class="lineno">  504</span> </div>
<div class="line"><a id="l00505" name="l00505"></a><span class="lineno">  505</span>        <span class="keywordflow">if</span> (tid == 0) {</div>
<div class="line"><a id="l00506" name="l00506"></a><span class="lineno">  506</span>            out[blockIdx.x] = localSum;</div>
<div class="line"><a id="l00507" name="l00507"></a><span class="lineno">  507</span>        }</div>
<div class="line"><a id="l00508" name="l00508"></a><span class="lineno">  508</span>    }</div>
<div class="line"><a id="l00509" name="l00509"></a><span class="lineno">  509</span> </div>
<div class="foldopen" id="foldopen00510" data-start="{" data-end="}">
<div class="line"><a id="l00510" name="l00510"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a51a5ff3c8cc2c3051fddf32de294b467">  510</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a51a5ff3c8cc2c3051fddf32de294b467">SummationExp</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keyword">const</span> <span class="keywordtype">size_t</span> sharedMemSize, <span class="keywordtype">float</span>* out,</div>
<div class="line"><a id="l00511" name="l00511"></a><span class="lineno">  511</span>                      <span class="keywordtype">float</span>* g_data,</div>
<div class="line"><a id="l00512" name="l00512"></a><span class="lineno">  512</span>                      <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l00513" name="l00513"></a><span class="lineno">  513</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(SummationExpKernel, gridDim, blockDim, sharedMemSize, out, g_data, n,</div>
<div class="line"><a id="l00514" name="l00514"></a><span class="lineno">  514</span>                                                offset);</div>
<div class="line"><a id="l00515" name="l00515"></a><span class="lineno">  515</span>    }</div>
</div>
<div class="line"><a id="l00516" name="l00516"></a><span class="lineno">  516</span> </div>
<div class="line"><a id="l00517" name="l00517"></a><span class="lineno">  517</span>    __global__ <span class="keywordtype">void</span> SoftmaxKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00518" name="l00518"></a><span class="lineno">  518</span>                                  <span class="keyword">const</span> <span class="keywordtype">float</span> exp_sum_of_input, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l00519" name="l00519"></a><span class="lineno">  519</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00520" name="l00520"></a><span class="lineno">  520</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00521" name="l00521"></a><span class="lineno">  521</span>            out[idx + offset] = __expf(in[idx + offset]) / exp_sum_of_input;</div>
<div class="line"><a id="l00522" name="l00522"></a><span class="lineno">  522</span>        }</div>
<div class="line"><a id="l00523" name="l00523"></a><span class="lineno">  523</span>    }</div>
<div class="line"><a id="l00524" name="l00524"></a><span class="lineno">  524</span> </div>
<div class="foldopen" id="foldopen00525" data-start="{" data-end="}">
<div class="line"><a id="l00525" name="l00525"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#adbafc409d57fa0a9d78ecac5bf7b10a3">  525</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#adbafc409d57fa0a9d78ecac5bf7b10a3">Softmax</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">float</span> exp_sum_of_input,</div>
<div class="line"><a id="l00526" name="l00526"></a><span class="lineno">  526</span>                 <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l00527" name="l00527"></a><span class="lineno">  527</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(SoftmaxKernel, gridDim, blockDim, 0, out, in, exp_sum_of_input, n,</div>
<div class="line"><a id="l00528" name="l00528"></a><span class="lineno">  528</span>                                                offset);</div>
<div class="line"><a id="l00529" name="l00529"></a><span class="lineno">  529</span>    }</div>
</div>
<div class="line"><a id="l00530" name="l00530"></a><span class="lineno">  530</span> </div>
<div class="line"><a id="l00531" name="l00531"></a><span class="lineno">  531</span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#adbafc409d57fa0a9d78ecac5bf7b10a3">Softmax</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00532" name="l00532"></a><span class="lineno">  532</span>                 <span class="keyword">const</span> std::vector&lt;float&gt;&amp; exp_sum_of_input,</div>
<div class="line"><a id="l00533" name="l00533"></a><span class="lineno">  533</span>                 <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset) {</div>
<div class="line"><a id="l00534" name="l00534"></a><span class="lineno">  534</span>        <span class="keywordflow">if</span> (exp_sum_of_input.size() != offset.size()) {</div>
<div class="line"><a id="l00535" name="l00535"></a><span class="lineno">  535</span>            <span class="keywordflow">throw</span> std::runtime_error(<span class="stringliteral">&quot;exp_sum_of_input and offset must have the same size&quot;</span>);</div>
<div class="line"><a id="l00536" name="l00536"></a><span class="lineno">  536</span>        }</div>
<div class="line"><a id="l00537" name="l00537"></a><span class="lineno">  537</span>        std::vector&lt;cudaStream_t&gt; streams;</div>
<div class="line"><a id="l00538" name="l00538"></a><span class="lineno">  538</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; exp_sum_of_input.size(); i++) {</div>
<div class="line"><a id="l00539" name="l00539"></a><span class="lineno">  539</span>            <span class="keyword">auto</span> stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l00540" name="l00540"></a><span class="lineno">  540</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(out, stream);</div>
<div class="line"><a id="l00541" name="l00541"></a><span class="lineno">  541</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(in, stream);</div>
<div class="line"><a id="l00542" name="l00542"></a><span class="lineno">  542</span>            streams.push_back(stream);</div>
<div class="line"><a id="l00543" name="l00543"></a><span class="lineno">  543</span>            SoftmaxKernel&lt;&lt;&lt;gridDim, blockDim, 0, stream&gt;&gt;&gt;(out, in, exp_sum_of_input[i], n, offset[i]);</div>
<div class="line"><a id="l00544" name="l00544"></a><span class="lineno">  544</span>        }</div>
<div class="line"><a id="l00545" name="l00545"></a><span class="lineno">  545</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l00546" name="l00546"></a><span class="lineno">  546</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(out, stream);</div>
<div class="line"><a id="l00547" name="l00547"></a><span class="lineno">  547</span>        }</div>
<div class="line"><a id="l00548" name="l00548"></a><span class="lineno">  548</span>    }</div>
<div class="line"><a id="l00549" name="l00549"></a><span class="lineno">  549</span> </div>
<div class="line"><a id="l00550" name="l00550"></a><span class="lineno">  550</span>    __global__ <span class="keywordtype">void</span> SoftmaxJacobianKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00551" name="l00551"></a><span class="lineno">  551</span>                                          <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_o = 0,</div>
<div class="line"><a id="l00552" name="l00552"></a><span class="lineno">  552</span>                                          <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_i = 0) {</div>
<div class="line"><a id="l00553" name="l00553"></a><span class="lineno">  553</span>        <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00554" name="l00554"></a><span class="lineno">  554</span>        <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idy = blockIdx.y * blockDim.y + threadIdx.y;</div>
<div class="line"><a id="l00555" name="l00555"></a><span class="lineno">  555</span>        <span class="keywordflow">if</span> (idx &gt;= n || idy &gt;= n) {</div>
<div class="line"><a id="l00556" name="l00556"></a><span class="lineno">  556</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l00557" name="l00557"></a><span class="lineno">  557</span>        }</div>
<div class="line"><a id="l00558" name="l00558"></a><span class="lineno">  558</span>        <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> <span class="keywordtype">id</span> = idx * n + idy;</div>
<div class="line"><a id="l00559" name="l00559"></a><span class="lineno">  559</span>        <span class="keywordflow">if</span> (idy == idx) {</div>
<div class="line"><a id="l00560" name="l00560"></a><span class="lineno">  560</span>            out[<span class="keywordtype">id</span> + offset_o] = in[idx + offset_i] * (1 - in[idx + offset_i]);</div>
<div class="line"><a id="l00561" name="l00561"></a><span class="lineno">  561</span>        }</div>
<div class="line"><a id="l00562" name="l00562"></a><span class="lineno">  562</span>        <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l00563" name="l00563"></a><span class="lineno">  563</span>            out[<span class="keywordtype">id</span> + offset_o] = -in[idx + offset_i] * in[idy + offset_i];</div>
<div class="line"><a id="l00564" name="l00564"></a><span class="lineno">  564</span>        }</div>
<div class="line"><a id="l00565" name="l00565"></a><span class="lineno">  565</span>    }</div>
<div class="line"><a id="l00566" name="l00566"></a><span class="lineno">  566</span> </div>
<div class="foldopen" id="foldopen00567" data-start="{" data-end="}">
<div class="line"><a id="l00567" name="l00567"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a4375738c83ef892783abc210578e5b39">  567</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a4375738c83ef892783abc210578e5b39">SoftmaxJacobian</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00568" name="l00568"></a><span class="lineno">  568</span>                         <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00569" name="l00569"></a><span class="lineno">  569</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(SoftmaxJacobianKernel, gridDim, blockDim, 0, out, in, n, 0, 0);</div>
<div class="line"><a id="l00570" name="l00570"></a><span class="lineno">  570</span>    }</div>
</div>
<div class="line"><a id="l00571" name="l00571"></a><span class="lineno">  571</span> </div>
<div class="line"><a id="l00572" name="l00572"></a><span class="lineno">  572</span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a4375738c83ef892783abc210578e5b39">SoftmaxJacobian</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l00573" name="l00573"></a><span class="lineno">  573</span>                         <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_o,</div>
<div class="line"><a id="l00574" name="l00574"></a><span class="lineno">  574</span>                         <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_i) {</div>
<div class="line"><a id="l00575" name="l00575"></a><span class="lineno">  575</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitParallel(SoftmaxJacobianKernel, gridDim, blockDim, 0, out, in, offset_o,</div>
<div class="line"><a id="l00576" name="l00576"></a><span class="lineno">  576</span>                                                        offset_i, n);</div>
<div class="line"><a id="l00577" name="l00577"></a><span class="lineno">  577</span>    }</div>
<div class="line"><a id="l00578" name="l00578"></a><span class="lineno">  578</span> </div>
<div class="line"><a id="l00579" name="l00579"></a><span class="lineno">  579</span>    __global__ <span class="keywordtype">void</span> MeanSquaredErrorKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* predict, <span class="keyword">const</span> <span class="keywordtype">float</span>* real,</div>
<div class="line"><a id="l00580" name="l00580"></a><span class="lineno">  580</span>                                           <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00581" name="l00581"></a><span class="lineno">  581</span>        <span class="keyword">extern</span> __shared__ <span class="keywordtype">float</span> smem[];</div>
<div class="line"><a id="l00582" name="l00582"></a><span class="lineno">  582</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00583" name="l00583"></a><span class="lineno">  583</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> tid = threadIdx.x;</div>
<div class="line"><a id="l00584" name="l00584"></a><span class="lineno">  584</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> warpIdx = tid / WARP_SIZE;</div>
<div class="line"><a id="l00585" name="l00585"></a><span class="lineno">  585</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> laneIdx = tid % WARP_SIZE;</div>
<div class="line"><a id="l00586" name="l00586"></a><span class="lineno">  586</span>        <span class="keywordtype">float</span> localSum = 0.0f;</div>
<div class="line"><a id="l00587" name="l00587"></a><span class="lineno">  587</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00588" name="l00588"></a><span class="lineno">  588</span>            localSum = (predict[idx] - real[idx]) * (predict[idx] - real[idx]) / (float)n;</div>
<div class="line"><a id="l00589" name="l00589"></a><span class="lineno">  589</span>        }</div>
<div class="line"><a id="l00590" name="l00590"></a><span class="lineno">  590</span>        <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l00591" name="l00591"></a><span class="lineno">  591</span>            localSum = 0.0f;</div>
<div class="line"><a id="l00592" name="l00592"></a><span class="lineno">  592</span>        }</div>
<div class="line"><a id="l00593" name="l00593"></a><span class="lineno">  593</span>        localSum = warpReduce(localSum);</div>
<div class="line"><a id="l00594" name="l00594"></a><span class="lineno">  594</span>        __syncthreads();</div>
<div class="line"><a id="l00595" name="l00595"></a><span class="lineno">  595</span> </div>
<div class="line"><a id="l00596" name="l00596"></a><span class="lineno">  596</span>        <span class="keywordflow">if</span> (laneIdx == 0) {</div>
<div class="line"><a id="l00597" name="l00597"></a><span class="lineno">  597</span>            smem[warpIdx] = localSum;</div>
<div class="line"><a id="l00598" name="l00598"></a><span class="lineno">  598</span>        }</div>
<div class="line"><a id="l00599" name="l00599"></a><span class="lineno">  599</span>        __syncthreads();</div>
<div class="line"><a id="l00600" name="l00600"></a><span class="lineno">  600</span> </div>
<div class="line"><a id="l00601" name="l00601"></a><span class="lineno">  601</span>        localSum = (tid &lt; blockDim.x / WARP_SIZE) ? smem[laneIdx] : 0.0f;</div>
<div class="line"><a id="l00602" name="l00602"></a><span class="lineno">  602</span> </div>
<div class="line"><a id="l00603" name="l00603"></a><span class="lineno">  603</span>        __syncthreads();</div>
<div class="line"><a id="l00604" name="l00604"></a><span class="lineno">  604</span> </div>
<div class="line"><a id="l00605" name="l00605"></a><span class="lineno">  605</span>        <span class="keywordflow">if</span> (warpIdx == 0) {</div>
<div class="line"><a id="l00606" name="l00606"></a><span class="lineno">  606</span>            localSum = warpReduce(localSum);</div>
<div class="line"><a id="l00607" name="l00607"></a><span class="lineno">  607</span>        }</div>
<div class="line"><a id="l00608" name="l00608"></a><span class="lineno">  608</span>        __syncthreads();</div>
<div class="line"><a id="l00609" name="l00609"></a><span class="lineno">  609</span> </div>
<div class="line"><a id="l00610" name="l00610"></a><span class="lineno">  610</span>        <span class="keywordflow">if</span> (tid == 0) {</div>
<div class="line"><a id="l00611" name="l00611"></a><span class="lineno">  611</span>            out[blockIdx.x] = localSum;</div>
<div class="line"><a id="l00612" name="l00612"></a><span class="lineno">  612</span>        }</div>
<div class="line"><a id="l00613" name="l00613"></a><span class="lineno">  613</span>    }</div>
<div class="line"><a id="l00614" name="l00614"></a><span class="lineno">  614</span> </div>
<div class="foldopen" id="foldopen00615" data-start="{" data-end="}">
<div class="line"><a id="l00615" name="l00615"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#af76ce6a930db4def5ceb51350af72f3c">  615</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#af76ce6a930db4def5ceb51350af72f3c">MeanSquaredError</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keyword">const</span> <span class="keywordtype">size_t</span> sharedMemSize, <span class="keywordtype">float</span>* out,</div>
<div class="line"><a id="l00616" name="l00616"></a><span class="lineno">  616</span>                          <span class="keywordtype">float</span>* predict, <span class="keywordtype">float</span>* real, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00617" name="l00617"></a><span class="lineno">  617</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(MeanSquaredErrorKernel, gridDim, blockDim, sharedMemSize, out, predict,</div>
<div class="line"><a id="l00618" name="l00618"></a><span class="lineno">  618</span>                                                real, n);</div>
<div class="line"><a id="l00619" name="l00619"></a><span class="lineno">  619</span>    }</div>
</div>
<div class="line"><a id="l00620" name="l00620"></a><span class="lineno">  620</span> </div>
<div class="line"><a id="l00621" name="l00621"></a><span class="lineno">  621</span>    __global__ <span class="keywordtype">void</span> MSEBackwardKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* predict,</div>
<div class="line"><a id="l00622" name="l00622"></a><span class="lineno">  622</span>                                      <span class="keyword">const</span> <span class="keywordtype">float</span>* real, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00623" name="l00623"></a><span class="lineno">  623</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00624" name="l00624"></a><span class="lineno">  624</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00625" name="l00625"></a><span class="lineno">  625</span>            out[idx] = 2 * (predict[idx] - real[idx]) / (<span class="keywordtype">float</span>)n;</div>
<div class="line"><a id="l00626" name="l00626"></a><span class="lineno">  626</span>        }</div>
<div class="line"><a id="l00627" name="l00627"></a><span class="lineno">  627</span>    }</div>
<div class="line"><a id="l00628" name="l00628"></a><span class="lineno">  628</span> </div>
<div class="foldopen" id="foldopen00629" data-start="{" data-end="}">
<div class="line"><a id="l00629" name="l00629"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#ae77920db6adf79a17dbfb1dbf1ab5656">  629</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#ae77920db6adf79a17dbfb1dbf1ab5656">MSEBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* predict,</div>
<div class="line"><a id="l00630" name="l00630"></a><span class="lineno">  630</span>                     <span class="keywordtype">float</span>* real, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00631" name="l00631"></a><span class="lineno">  631</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(MSEBackwardKernel, gridDim, blockDim, 0, out, predict, real, n);</div>
<div class="line"><a id="l00632" name="l00632"></a><span class="lineno">  632</span>    }</div>
</div>
<div class="line"><a id="l00633" name="l00633"></a><span class="lineno">  633</span> </div>
<div class="line"><a id="l00634" name="l00634"></a><span class="lineno">  634</span>    __global__ <span class="keywordtype">void</span> StochasticGradientDescentKernel(<span class="keywordtype">float</span>* data, <span class="keyword">const</span> <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> lr,</div>
<div class="line"><a id="l00635" name="l00635"></a><span class="lineno">  635</span>                                                    <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00636" name="l00636"></a><span class="lineno">  636</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00637" name="l00637"></a><span class="lineno">  637</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00638" name="l00638"></a><span class="lineno">  638</span>            data[idx] -= lr * grad[idx];</div>
<div class="line"><a id="l00639" name="l00639"></a><span class="lineno">  639</span>        }</div>
<div class="line"><a id="l00640" name="l00640"></a><span class="lineno">  640</span>    }</div>
<div class="line"><a id="l00641" name="l00641"></a><span class="lineno">  641</span> </div>
<div class="foldopen" id="foldopen00642" data-start="{" data-end="}">
<div class="line"><a id="l00642" name="l00642"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aeec286d5351eee7061e151470adb4eef">  642</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aeec286d5351eee7061e151470adb4eef">StochasticGradientDescent</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* grad,</div>
<div class="line"><a id="l00643" name="l00643"></a><span class="lineno">  643</span>                                   <span class="keyword">const</span> <span class="keywordtype">float</span> lr, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00644" name="l00644"></a><span class="lineno">  644</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(StochasticGradientDescentKernel, gridDim, blockDim, 0, data, grad, lr,</div>
<div class="line"><a id="l00645" name="l00645"></a><span class="lineno">  645</span>                                                n);</div>
<div class="line"><a id="l00646" name="l00646"></a><span class="lineno">  646</span>    }</div>
</div>
<div class="line"><a id="l00647" name="l00647"></a><span class="lineno">  647</span> </div>
<div class="line"><a id="l00648" name="l00648"></a><span class="lineno">  648</span>    __global__ <span class="keywordtype">void</span> BinaryCrossEntropyKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* predict, <span class="keyword">const</span> <span class="keywordtype">float</span>* real,</div>
<div class="line"><a id="l00649" name="l00649"></a><span class="lineno">  649</span>                                             <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00650" name="l00650"></a><span class="lineno">  650</span>        <span class="keyword">extern</span> __shared__ <span class="keywordtype">float</span> smem[];</div>
<div class="line"><a id="l00651" name="l00651"></a><span class="lineno">  651</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00652" name="l00652"></a><span class="lineno">  652</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> tid = threadIdx.x;</div>
<div class="line"><a id="l00653" name="l00653"></a><span class="lineno">  653</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> warpIdx = tid / WARP_SIZE;</div>
<div class="line"><a id="l00654" name="l00654"></a><span class="lineno">  654</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> laneIdx = tid % WARP_SIZE;</div>
<div class="line"><a id="l00655" name="l00655"></a><span class="lineno">  655</span>        <span class="keywordtype">float</span> localSum = 0.0f;</div>
<div class="line"><a id="l00656" name="l00656"></a><span class="lineno">  656</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00657" name="l00657"></a><span class="lineno">  657</span>            localSum = (-real[idx] * __logf(predict[idx]) - (1 - real[idx]) *</div>
<div class="line"><a id="l00658" name="l00658"></a><span class="lineno">  658</span>                __logf(1 - predict[idx])) / (<span class="keywordtype">float</span>)n;</div>
<div class="line"><a id="l00659" name="l00659"></a><span class="lineno">  659</span>        }</div>
<div class="line"><a id="l00660" name="l00660"></a><span class="lineno">  660</span>        <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l00661" name="l00661"></a><span class="lineno">  661</span>            localSum = 0;</div>
<div class="line"><a id="l00662" name="l00662"></a><span class="lineno">  662</span>        }</div>
<div class="line"><a id="l00663" name="l00663"></a><span class="lineno">  663</span>        __syncthreads();</div>
<div class="line"><a id="l00664" name="l00664"></a><span class="lineno">  664</span> </div>
<div class="line"><a id="l00665" name="l00665"></a><span class="lineno">  665</span>        localSum = warpReduce(localSum);</div>
<div class="line"><a id="l00666" name="l00666"></a><span class="lineno">  666</span>        __syncthreads();</div>
<div class="line"><a id="l00667" name="l00667"></a><span class="lineno">  667</span> </div>
<div class="line"><a id="l00668" name="l00668"></a><span class="lineno">  668</span>        <span class="keywordflow">if</span> (laneIdx == 0) {</div>
<div class="line"><a id="l00669" name="l00669"></a><span class="lineno">  669</span>            smem[warpIdx] = localSum;</div>
<div class="line"><a id="l00670" name="l00670"></a><span class="lineno">  670</span>        }</div>
<div class="line"><a id="l00671" name="l00671"></a><span class="lineno">  671</span>        __syncthreads();</div>
<div class="line"><a id="l00672" name="l00672"></a><span class="lineno">  672</span> </div>
<div class="line"><a id="l00673" name="l00673"></a><span class="lineno">  673</span>        localSum = (tid &lt; blockDim.x / WARP_SIZE) ? smem[laneIdx] : 0.0f;</div>
<div class="line"><a id="l00674" name="l00674"></a><span class="lineno">  674</span>        __syncthreads();</div>
<div class="line"><a id="l00675" name="l00675"></a><span class="lineno">  675</span> </div>
<div class="line"><a id="l00676" name="l00676"></a><span class="lineno">  676</span>        <span class="keywordflow">if</span> (warpIdx == 0) {</div>
<div class="line"><a id="l00677" name="l00677"></a><span class="lineno">  677</span>            localSum = warpReduce(localSum);</div>
<div class="line"><a id="l00678" name="l00678"></a><span class="lineno">  678</span>        }</div>
<div class="line"><a id="l00679" name="l00679"></a><span class="lineno">  679</span>        __syncthreads();</div>
<div class="line"><a id="l00680" name="l00680"></a><span class="lineno">  680</span> </div>
<div class="line"><a id="l00681" name="l00681"></a><span class="lineno">  681</span>        <span class="keywordflow">if</span> (tid == 0) {</div>
<div class="line"><a id="l00682" name="l00682"></a><span class="lineno">  682</span>            out[blockIdx.x] = localSum;</div>
<div class="line"><a id="l00683" name="l00683"></a><span class="lineno">  683</span>        }</div>
<div class="line"><a id="l00684" name="l00684"></a><span class="lineno">  684</span>    }</div>
<div class="line"><a id="l00685" name="l00685"></a><span class="lineno">  685</span> </div>
<div class="foldopen" id="foldopen00686" data-start="{" data-end="}">
<div class="line"><a id="l00686" name="l00686"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#abf927faf0950fbc215564c67b8ac57be">  686</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#abf927faf0950fbc215564c67b8ac57be">BinaryCrossEntropy</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keyword">const</span> <span class="keywordtype">size_t</span> sharedMemSize, <span class="keywordtype">float</span>* out,</div>
<div class="line"><a id="l00687" name="l00687"></a><span class="lineno">  687</span>                            <span class="keywordtype">float</span>* predict, <span class="keywordtype">float</span>* real, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00688" name="l00688"></a><span class="lineno">  688</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(BinaryCrossEntropyKernel, gridDim, blockDim, sharedMemSize, out,</div>
<div class="line"><a id="l00689" name="l00689"></a><span class="lineno">  689</span>                                                predict, real, n);</div>
<div class="line"><a id="l00690" name="l00690"></a><span class="lineno">  690</span>    }</div>
</div>
<div class="line"><a id="l00691" name="l00691"></a><span class="lineno">  691</span> </div>
<div class="line"><a id="l00692" name="l00692"></a><span class="lineno">  692</span>    __global__ <span class="keywordtype">void</span> BCEBackwardKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* predict,</div>
<div class="line"><a id="l00693" name="l00693"></a><span class="lineno">  693</span>                                      <span class="keyword">const</span> <span class="keywordtype">float</span>* real, <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00694" name="l00694"></a><span class="lineno">  694</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00695" name="l00695"></a><span class="lineno">  695</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00696" name="l00696"></a><span class="lineno">  696</span>            out[idx] = ((predict[idx] - real[idx]) / (</div>
<div class="line"><a id="l00697" name="l00697"></a><span class="lineno">  697</span>                predict[idx] * (1 - predict[idx]))) / (<span class="keywordtype">float</span>)n;</div>
<div class="line"><a id="l00698" name="l00698"></a><span class="lineno">  698</span>        }</div>
<div class="line"><a id="l00699" name="l00699"></a><span class="lineno">  699</span>    }</div>
<div class="line"><a id="l00700" name="l00700"></a><span class="lineno">  700</span> </div>
<div class="foldopen" id="foldopen00701" data-start="{" data-end="}">
<div class="line"><a id="l00701" name="l00701"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a1fc3d553947a5cad87f29989f9d9465d">  701</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a1fc3d553947a5cad87f29989f9d9465d">BCEBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* predict,</div>
<div class="line"><a id="l00702" name="l00702"></a><span class="lineno">  702</span>                     <span class="keywordtype">float</span>* real, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00703" name="l00703"></a><span class="lineno">  703</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(BCEBackwardKernel, gridDim, blockDim, 0, out, predict, real, n);</div>
<div class="line"><a id="l00704" name="l00704"></a><span class="lineno">  704</span>    }</div>
</div>
<div class="line"><a id="l00705" name="l00705"></a><span class="lineno">  705</span> </div>
<div class="line"><a id="l00706" name="l00706"></a><span class="lineno">  706</span>    __global__ <span class="keywordtype">void</span> MomentumKernel(<span class="keywordtype">float</span>* output, <span class="keyword">const</span> <span class="keywordtype">float</span>* grad,</div>
<div class="line"><a id="l00707" name="l00707"></a><span class="lineno">  707</span>                                   <span class="keyword">const</span> <span class="keywordtype">float</span>* velocity, <span class="keyword">const</span> <span class="keywordtype">float</span> beta,</div>
<div class="line"><a id="l00708" name="l00708"></a><span class="lineno">  708</span>                                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00709" name="l00709"></a><span class="lineno">  709</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00710" name="l00710"></a><span class="lineno">  710</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l00711" name="l00711"></a><span class="lineno">  711</span>            output[idx] = velocity[idx] * beta + grad[idx] * (1 - beta);</div>
<div class="line"><a id="l00712" name="l00712"></a><span class="lineno">  712</span>        }</div>
<div class="line"><a id="l00713" name="l00713"></a><span class="lineno">  713</span>    }</div>
<div class="line"><a id="l00714" name="l00714"></a><span class="lineno">  714</span> </div>
<div class="foldopen" id="foldopen00715" data-start="{" data-end="}">
<div class="line"><a id="l00715" name="l00715"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a273ef3023442a864f1028becaf236bae">  715</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a273ef3023442a864f1028becaf236bae">Momentum</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* output, <span class="keywordtype">float</span>* grad, <span class="keywordtype">float</span>* velocity,</div>
<div class="line"><a id="l00716" name="l00716"></a><span class="lineno">  716</span>                  <span class="keyword">const</span> <span class="keywordtype">float</span> beta, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00717" name="l00717"></a><span class="lineno">  717</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(MomentumKernel, gridDim, blockDim, 0, output, grad, velocity, beta, n);</div>
<div class="line"><a id="l00718" name="l00718"></a><span class="lineno">  718</span>    }</div>
</div>
<div class="line"><a id="l00719" name="l00719"></a><span class="lineno">  719</span> </div>
<div class="line"><a id="l00720" name="l00720"></a><span class="lineno">  720</span>    __global__ <span class="keywordtype">void</span> AdaGradKernel(<span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* G, <span class="keyword">const</span> <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> lr, <span class="keyword">const</span> <span class="keywordtype">float</span> eps,</div>
<div class="line"><a id="l00721" name="l00721"></a><span class="lineno">  721</span>                                  <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00722" name="l00722"></a><span class="lineno">  722</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00723" name="l00723"></a><span class="lineno">  723</span>        <span class="keywordflow">if</span> (idx &gt;= n) {</div>
<div class="line"><a id="l00724" name="l00724"></a><span class="lineno">  724</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l00725" name="l00725"></a><span class="lineno">  725</span>        }</div>
<div class="line"><a id="l00726" name="l00726"></a><span class="lineno">  726</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> temp = G[idx] + grad[idx] * grad[idx];</div>
<div class="line"><a id="l00727" name="l00727"></a><span class="lineno">  727</span>        data[idx] -= lr * grad[idx] / (sqrtf(temp) + eps);</div>
<div class="line"><a id="l00728" name="l00728"></a><span class="lineno">  728</span>        G[idx] = temp;</div>
<div class="line"><a id="l00729" name="l00729"></a><span class="lineno">  729</span>    }</div>
<div class="line"><a id="l00730" name="l00730"></a><span class="lineno">  730</span> </div>
<div class="foldopen" id="foldopen00731" data-start="{" data-end="}">
<div class="line"><a id="l00731" name="l00731"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a1e915bd4a354938d8bc2d09be00eae76">  731</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a1e915bd4a354938d8bc2d09be00eae76">AdaGrad</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* G, <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> lr,</div>
<div class="line"><a id="l00732" name="l00732"></a><span class="lineno">  732</span>                 <span class="keyword">const</span> <span class="keywordtype">float</span> eps, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00733" name="l00733"></a><span class="lineno">  733</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitDualOut(AdaGradKernel, gridDim, blockDim, 0, data, G, grad, lr, eps, n);</div>
<div class="line"><a id="l00734" name="l00734"></a><span class="lineno">  734</span>    }</div>
</div>
<div class="line"><a id="l00735" name="l00735"></a><span class="lineno">  735</span> </div>
<div class="line"><a id="l00736" name="l00736"></a><span class="lineno">  736</span>    __global__ <span class="keywordtype">void</span> RMSpropKernel(<span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* v, <span class="keyword">const</span> <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> lr, <span class="keyword">const</span> <span class="keywordtype">float</span> beta,</div>
<div class="line"><a id="l00737" name="l00737"></a><span class="lineno">  737</span>                                  <span class="keyword">const</span> <span class="keywordtype">float</span> eps, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00738" name="l00738"></a><span class="lineno">  738</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00739" name="l00739"></a><span class="lineno">  739</span>        <span class="keywordflow">if</span> (idx &gt;= n) {</div>
<div class="line"><a id="l00740" name="l00740"></a><span class="lineno">  740</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l00741" name="l00741"></a><span class="lineno">  741</span>        }</div>
<div class="line"><a id="l00742" name="l00742"></a><span class="lineno">  742</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> temp = v[idx] * beta + grad[idx] * grad[idx] * (1 - beta);</div>
<div class="line"><a id="l00743" name="l00743"></a><span class="lineno">  743</span>        data[idx] -= lr * grad[idx] / (sqrtf(temp) + eps);</div>
<div class="line"><a id="l00744" name="l00744"></a><span class="lineno">  744</span>        v[idx] = temp;</div>
<div class="line"><a id="l00745" name="l00745"></a><span class="lineno">  745</span>    }</div>
<div class="line"><a id="l00746" name="l00746"></a><span class="lineno">  746</span> </div>
<div class="foldopen" id="foldopen00747" data-start="{" data-end="}">
<div class="line"><a id="l00747" name="l00747"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aaf3c9cca114d003130ffa4354b4a24de">  747</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aaf3c9cca114d003130ffa4354b4a24de">RMSprop</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* v, <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> lr,</div>
<div class="line"><a id="l00748" name="l00748"></a><span class="lineno">  748</span>                 <span class="keyword">const</span> <span class="keywordtype">float</span> beta, <span class="keyword">const</span> <span class="keywordtype">float</span> eps, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00749" name="l00749"></a><span class="lineno">  749</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitDualOut(RMSpropKernel, gridDim, blockDim, 0, data, v, grad, lr, beta,</div>
<div class="line"><a id="l00750" name="l00750"></a><span class="lineno">  750</span>                                                       eps, n);</div>
<div class="line"><a id="l00751" name="l00751"></a><span class="lineno">  751</span>    }</div>
</div>
<div class="line"><a id="l00752" name="l00752"></a><span class="lineno">  752</span> </div>
<div class="line"><a id="l00753" name="l00753"></a><span class="lineno">  753</span>    __global__ <span class="keywordtype">void</span> AdamKernel(<span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* m, <span class="keywordtype">float</span>* v, <span class="keyword">const</span> <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> lr, <span class="keyword">const</span> <span class="keywordtype">float</span> beta1,</div>
<div class="line"><a id="l00754" name="l00754"></a><span class="lineno">  754</span>                               <span class="keyword">const</span> <span class="keywordtype">float</span> beta2, <span class="keyword">const</span> <span class="keywordtype">float</span> eps, <span class="keyword">const</span> <span class="keywordtype">int</span> t, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00755" name="l00755"></a><span class="lineno">  755</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00756" name="l00756"></a><span class="lineno">  756</span>        <span class="keywordflow">if</span> (idx &gt;= n) {</div>
<div class="line"><a id="l00757" name="l00757"></a><span class="lineno">  757</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l00758" name="l00758"></a><span class="lineno">  758</span>        }</div>
<div class="line"><a id="l00759" name="l00759"></a><span class="lineno">  759</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> m_temp = m[idx] * beta1 + grad[idx] * (1 - beta1);</div>
<div class="line"><a id="l00760" name="l00760"></a><span class="lineno">  760</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> v_temp = v[idx] * beta2 + grad[idx] * grad[idx] * (1 - beta2);</div>
<div class="line"><a id="l00761" name="l00761"></a><span class="lineno">  761</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> m_modified = m_temp / (1 - __powf(beta1, (<span class="keywordtype">float</span>)t));</div>
<div class="line"><a id="l00762" name="l00762"></a><span class="lineno">  762</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> v_modified = v_temp / (1 - __powf(beta2, (<span class="keywordtype">float</span>)t));</div>
<div class="line"><a id="l00763" name="l00763"></a><span class="lineno">  763</span>        data[idx] -= lr * m_modified / (sqrtf(v_modified) + eps);</div>
<div class="line"><a id="l00764" name="l00764"></a><span class="lineno">  764</span>        m[idx] = m_temp;</div>
<div class="line"><a id="l00765" name="l00765"></a><span class="lineno">  765</span>        v[idx] = v_temp;</div>
<div class="line"><a id="l00766" name="l00766"></a><span class="lineno">  766</span>    }</div>
<div class="line"><a id="l00767" name="l00767"></a><span class="lineno">  767</span> </div>
<div class="foldopen" id="foldopen00768" data-start="{" data-end="}">
<div class="line"><a id="l00768" name="l00768"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a2b9ab840eeb0e74f4b78277a046b3a07">  768</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a2b9ab840eeb0e74f4b78277a046b3a07">Adam</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* m, <span class="keywordtype">float</span>* v, <span class="keywordtype">float</span>* grad,</div>
<div class="line"><a id="l00769" name="l00769"></a><span class="lineno">  769</span>              <span class="keyword">const</span> <span class="keywordtype">float</span> lr, <span class="keyword">const</span> <span class="keywordtype">float</span> beta1, <span class="keyword">const</span> <span class="keywordtype">float</span> beta2, <span class="keyword">const</span> <span class="keywordtype">float</span> eps, <span class="keyword">const</span> <span class="keywordtype">int</span> t,</div>
<div class="line"><a id="l00770" name="l00770"></a><span class="lineno">  770</span>              <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00771" name="l00771"></a><span class="lineno">  771</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitTripleOut(AdamKernel, gridDim, blockDim, 0, data, m, v, grad, lr, beta1,</div>
<div class="line"><a id="l00772" name="l00772"></a><span class="lineno">  772</span>                                                         beta2, eps, t, n);</div>
<div class="line"><a id="l00773" name="l00773"></a><span class="lineno">  773</span>    }</div>
</div>
<div class="line"><a id="l00774" name="l00774"></a><span class="lineno">  774</span> </div>
<div class="line"><a id="l00775" name="l00775"></a><span class="lineno">  775</span>    __global__ <span class="keywordtype">void</span> NAdamKernel(<span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* m, <span class="keywordtype">float</span>* m_modified, <span class="keywordtype">float</span>* v, <span class="keyword">const</span> <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> lr,</div>
<div class="line"><a id="l00776" name="l00776"></a><span class="lineno">  776</span>                                <span class="keyword">const</span> <span class="keywordtype">float</span> beta1, <span class="keyword">const</span> <span class="keywordtype">float</span> beta2, <span class="keyword">const</span> <span class="keywordtype">float</span> eps, <span class="keyword">const</span> <span class="keywordtype">int</span> t,</div>
<div class="line"><a id="l00777" name="l00777"></a><span class="lineno">  777</span>                                <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00778" name="l00778"></a><span class="lineno">  778</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00779" name="l00779"></a><span class="lineno">  779</span>        <span class="keywordflow">if</span> (idx &gt;= n) {</div>
<div class="line"><a id="l00780" name="l00780"></a><span class="lineno">  780</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l00781" name="l00781"></a><span class="lineno">  781</span>        }</div>
<div class="line"><a id="l00782" name="l00782"></a><span class="lineno">  782</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> m_temp = m[idx] * beta1 + grad[idx] * (1 - beta1);</div>
<div class="line"><a id="l00783" name="l00783"></a><span class="lineno">  783</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> v_temp = v[idx] * beta2 + grad[idx] * grad[idx] * (1 - beta2);</div>
<div class="line"><a id="l00784" name="l00784"></a><span class="lineno">  784</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> m_temp_modified = m_temp / (1 - __powf(beta1, (<span class="keywordtype">float</span>)t));</div>
<div class="line"><a id="l00785" name="l00785"></a><span class="lineno">  785</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> v_modified = v_temp / (1 - __powf(beta2, (<span class="keywordtype">float</span>)t));</div>
<div class="line"><a id="l00786" name="l00786"></a><span class="lineno">  786</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> m_modified_minus_1 = m_modified[idx] * beta1 + grad[idx] * (1 - beta1);</div>
<div class="line"><a id="l00787" name="l00787"></a><span class="lineno">  787</span>        data[idx] -= lr * m_modified_minus_1 / (sqrtf(v_modified) + eps);</div>
<div class="line"><a id="l00788" name="l00788"></a><span class="lineno">  788</span>        m[idx] = m_temp;</div>
<div class="line"><a id="l00789" name="l00789"></a><span class="lineno">  789</span>        m_modified[idx] = m_temp_modified;</div>
<div class="line"><a id="l00790" name="l00790"></a><span class="lineno">  790</span>        v[idx] = v_temp;</div>
<div class="line"><a id="l00791" name="l00791"></a><span class="lineno">  791</span>    }</div>
<div class="line"><a id="l00792" name="l00792"></a><span class="lineno">  792</span> </div>
<div class="foldopen" id="foldopen00793" data-start="{" data-end="}">
<div class="line"><a id="l00793" name="l00793"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#ada94b8c5c6e6d72132face63a3305624">  793</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#ada94b8c5c6e6d72132face63a3305624">NAdam</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* m, <span class="keywordtype">float</span>* m_modified, <span class="keywordtype">float</span>* v,</div>
<div class="line"><a id="l00794" name="l00794"></a><span class="lineno">  794</span>               <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> lr, <span class="keyword">const</span> <span class="keywordtype">float</span> beta1, <span class="keyword">const</span> <span class="keywordtype">float</span> beta2, <span class="keyword">const</span> <span class="keywordtype">float</span> eps, <span class="keyword">const</span> <span class="keywordtype">int</span> t,</div>
<div class="line"><a id="l00795" name="l00795"></a><span class="lineno">  795</span>               <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00796" name="l00796"></a><span class="lineno">  796</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitQuadOut(NAdamKernel, gridDim, blockDim, 0, data, m, m_modified, v, grad,</div>
<div class="line"><a id="l00797" name="l00797"></a><span class="lineno">  797</span>                                                       lr, beta1, beta2, eps, t, n);</div>
<div class="line"><a id="l00798" name="l00798"></a><span class="lineno">  798</span>    }</div>
</div>
<div class="line"><a id="l00799" name="l00799"></a><span class="lineno">  799</span> </div>
<div class="line"><a id="l00800" name="l00800"></a><span class="lineno">  800</span>    __global__ <span class="keywordtype">void</span> AdaDeltaKernel(<span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* acc_delta, <span class="keywordtype">float</span>* acc_grad, <span class="keyword">const</span> <span class="keywordtype">float</span>* grad,</div>
<div class="line"><a id="l00801" name="l00801"></a><span class="lineno">  801</span>                                   <span class="keyword">const</span> <span class="keywordtype">float</span> rho, <span class="keyword">const</span> <span class="keywordtype">float</span> eps,</div>
<div class="line"><a id="l00802" name="l00802"></a><span class="lineno">  802</span>                                   <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00803" name="l00803"></a><span class="lineno">  803</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00804" name="l00804"></a><span class="lineno">  804</span>        <span class="keywordflow">if</span> (idx &gt;= n) {</div>
<div class="line"><a id="l00805" name="l00805"></a><span class="lineno">  805</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l00806" name="l00806"></a><span class="lineno">  806</span>        }</div>
<div class="line"><a id="l00807" name="l00807"></a><span class="lineno">  807</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> delta_acc_grad_temp = acc_grad[idx] * rho + grad[idx] * grad[idx] * (1 - rho);</div>
<div class="line"><a id="l00808" name="l00808"></a><span class="lineno">  808</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> delta_theta = -grad[idx] * sqrtf(acc_delta[idx] + eps) / sqrtf(delta_acc_grad_temp + eps);</div>
<div class="line"><a id="l00809" name="l00809"></a><span class="lineno">  809</span>        data[idx] += delta_theta;</div>
<div class="line"><a id="l00810" name="l00810"></a><span class="lineno">  810</span>        <span class="keyword">const</span> <span class="keywordtype">float</span> delta_acc_temp = acc_delta[idx] * rho + delta_theta * delta_theta * (1 - rho);</div>
<div class="line"><a id="l00811" name="l00811"></a><span class="lineno">  811</span>        acc_delta[idx] = delta_acc_temp;</div>
<div class="line"><a id="l00812" name="l00812"></a><span class="lineno">  812</span>        acc_grad[idx] = delta_acc_grad_temp;</div>
<div class="line"><a id="l00813" name="l00813"></a><span class="lineno">  813</span>    }</div>
<div class="line"><a id="l00814" name="l00814"></a><span class="lineno">  814</span> </div>
<div class="foldopen" id="foldopen00815" data-start="{" data-end="}">
<div class="line"><a id="l00815" name="l00815"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a1f71726879c2d6a9d790522cdc1576e1">  815</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a1f71726879c2d6a9d790522cdc1576e1">AdaDelta</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* data, <span class="keywordtype">float</span>* acc_delta, <span class="keywordtype">float</span>* acc_grad,</div>
<div class="line"><a id="l00816" name="l00816"></a><span class="lineno">  816</span>                  <span class="keywordtype">float</span>* grad, <span class="keyword">const</span> <span class="keywordtype">float</span> rho, <span class="keyword">const</span> <span class="keywordtype">float</span> eps, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l00817" name="l00817"></a><span class="lineno">  817</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitTripleOut(AdaDeltaKernel, gridDim, blockDim, 0, data, acc_delta,</div>
<div class="line"><a id="l00818" name="l00818"></a><span class="lineno">  818</span>                                                         acc_grad, grad, rho, eps, n);</div>
<div class="line"><a id="l00819" name="l00819"></a><span class="lineno">  819</span>    }</div>
</div>
<div class="line"><a id="l00820" name="l00820"></a><span class="lineno">  820</span> </div>
<div class="line"><a id="l00821" name="l00821"></a><span class="lineno">  821</span>    __global__ <span class="keywordtype">void</span> GeneralMatrixMulTensorKernel(<span class="keywordtype">float</span>* C, <span class="keyword">const</span> half* A, <span class="keyword">const</span> half* B, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> m,</div>
<div class="line"><a id="l00822" name="l00822"></a><span class="lineno">  822</span>                                                 <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> k,</div>
<div class="line"><a id="l00823" name="l00823"></a><span class="lineno">  823</span>                                                 <span class="keywordtype">size_t</span> offset_c = 0,</div>
<div class="line"><a id="l00824" name="l00824"></a><span class="lineno">  824</span>                                                 <span class="keywordtype">size_t</span> offset_a = 0,</div>
<div class="line"><a id="l00825" name="l00825"></a><span class="lineno">  825</span>                                                 <span class="keywordtype">size_t</span> offset_b = 0) {</div>
<div class="line"><a id="l00826" name="l00826"></a><span class="lineno">  826</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00827" name="l00827"></a><span class="lineno">  827</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> warpIdx = idx / warpSize;</div>
<div class="line"><a id="l00828" name="l00828"></a><span class="lineno">  828</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> blockM = m / MMA;</div>
<div class="line"><a id="l00829" name="l00829"></a><span class="lineno">  829</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> blockN = n / MMA;</div>
<div class="line"><a id="l00830" name="l00830"></a><span class="lineno">  830</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> rowA = warpIdx / blockN;</div>
<div class="line"><a id="l00831" name="l00831"></a><span class="lineno">  831</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> colB = warpIdx % blockN;</div>
<div class="line"><a id="l00832" name="l00832"></a><span class="lineno">  832</span>        nvcuda::wmma::fragment&lt;nvcuda::wmma::matrix_a, MMA, MMA, MMA, half, nvcuda::wmma::row_major&gt; A_frag;</div>
<div class="line"><a id="l00833" name="l00833"></a><span class="lineno">  833</span>        nvcuda::wmma::fragment&lt;nvcuda::wmma::matrix_b, MMA, MMA, MMA, half, nvcuda::wmma::row_major&gt; B_frag;</div>
<div class="line"><a id="l00834" name="l00834"></a><span class="lineno">  834</span>        nvcuda::wmma::fragment&lt;nvcuda::wmma::accumulator, MMA, MMA, MMA, float&gt; C_frag;</div>
<div class="line"><a id="l00835" name="l00835"></a><span class="lineno">  835</span>        fill_fragment(C_frag, 0.0f);</div>
<div class="line"><a id="l00836" name="l00836"></a><span class="lineno">  836</span>        <span class="keywordflow">if</span> (rowA &lt; blockM &amp;&amp; colB &lt; blockN) {</div>
<div class="line"><a id="l00837" name="l00837"></a><span class="lineno">  837</span>            <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i &lt; k / MMA; i++) {</div>
<div class="line"><a id="l00838" name="l00838"></a><span class="lineno">  838</span>                load_matrix_sync(A_frag, A + offset_a + rowA * k * MMA + i * MMA, k);</div>
<div class="line"><a id="l00839" name="l00839"></a><span class="lineno">  839</span>                load_matrix_sync(B_frag, B + offset_b + colB * MMA + i * n * MMA, n);</div>
<div class="line"><a id="l00840" name="l00840"></a><span class="lineno">  840</span>                mma_sync(C_frag, A_frag, B_frag, C_frag);</div>
<div class="line"><a id="l00841" name="l00841"></a><span class="lineno">  841</span>            }</div>
<div class="line"><a id="l00842" name="l00842"></a><span class="lineno">  842</span>            store_matrix_sync(C + offset_c + rowA * n * MMA + colB * MMA, C_frag, n, nvcuda::wmma::mem_row_major);</div>
<div class="line"><a id="l00843" name="l00843"></a><span class="lineno">  843</span>        }</div>
<div class="line"><a id="l00844" name="l00844"></a><span class="lineno">  844</span>    }</div>
<div class="line"><a id="l00845" name="l00845"></a><span class="lineno">  845</span> </div>
<div class="line"><a id="l00846" name="l00846"></a><span class="lineno">  846</span>    __global__ <span class="keywordtype">void</span> Padding(half* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> M, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> N,</div>
<div class="line"><a id="l00847" name="l00847"></a><span class="lineno">  847</span>                            <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> m, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keywordtype">size_t</span> offset_o = 0,</div>
<div class="line"><a id="l00848" name="l00848"></a><span class="lineno">  848</span>                            <span class="keywordtype">size_t</span> offset_i = 0) {</div>
<div class="line"><a id="l00849" name="l00849"></a><span class="lineno">  849</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00850" name="l00850"></a><span class="lineno">  850</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> row = idx / n;</div>
<div class="line"><a id="l00851" name="l00851"></a><span class="lineno">  851</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> col = idx % n;</div>
<div class="line"><a id="l00852" name="l00852"></a><span class="lineno">  852</span>        <span class="keywordflow">if</span> (row &lt; m &amp;&amp; col &lt; n) {</div>
<div class="line"><a id="l00853" name="l00853"></a><span class="lineno">  853</span>            <span class="keywordflow">if</span> (row &lt; M &amp;&amp; col &lt; N) {</div>
<div class="line"><a id="l00854" name="l00854"></a><span class="lineno">  854</span>                out[row * n + col + offset_o] = __float2half(in[row * N + col + offset_i]);</div>
<div class="line"><a id="l00855" name="l00855"></a><span class="lineno">  855</span>            }</div>
<div class="line"><a id="l00856" name="l00856"></a><span class="lineno">  856</span>            <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l00857" name="l00857"></a><span class="lineno">  857</span>                out[row * n + col + offset_o] = __float2half(0.0f);</div>
<div class="line"><a id="l00858" name="l00858"></a><span class="lineno">  858</span>            }</div>
<div class="line"><a id="l00859" name="l00859"></a><span class="lineno">  859</span>        }</div>
<div class="line"><a id="l00860" name="l00860"></a><span class="lineno">  860</span>    }</div>
<div class="line"><a id="l00861" name="l00861"></a><span class="lineno">  861</span> </div>
<div class="line"><a id="l00862" name="l00862"></a><span class="lineno">  862</span>    __global__ <span class="keywordtype">void</span> Cutting(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> M, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> N,</div>
<div class="line"><a id="l00863" name="l00863"></a><span class="lineno">  863</span>                            <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> m, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keywordtype">size_t</span> offset_o = 0,</div>
<div class="line"><a id="l00864" name="l00864"></a><span class="lineno">  864</span>                            <span class="keywordtype">size_t</span> offset_i = 0) {</div>
<div class="line"><a id="l00865" name="l00865"></a><span class="lineno">  865</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00866" name="l00866"></a><span class="lineno">  866</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> row = idx / n;</div>
<div class="line"><a id="l00867" name="l00867"></a><span class="lineno">  867</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> col = idx % n;</div>
<div class="line"><a id="l00868" name="l00868"></a><span class="lineno">  868</span>        <span class="keywordflow">if</span> (row &lt; M &amp;&amp; col &lt; N) {</div>
<div class="line"><a id="l00869" name="l00869"></a><span class="lineno">  869</span>            out[row * N + col + offset_o] = in[row * n + col + offset_i];</div>
<div class="line"><a id="l00870" name="l00870"></a><span class="lineno">  870</span>        }</div>
<div class="line"><a id="l00871" name="l00871"></a><span class="lineno">  871</span>    }</div>
<div class="line"><a id="l00872" name="l00872"></a><span class="lineno">  872</span> </div>
<div class="line"><a id="l00873" name="l00873"></a><span class="lineno">  873</span>    __global__ <span class="keywordtype">void</span> CuttingAndCompress(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> M,</div>
<div class="line"><a id="l00874" name="l00874"></a><span class="lineno">  874</span>                                       <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> N,</div>
<div class="line"><a id="l00875" name="l00875"></a><span class="lineno">  875</span>                                       <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> m, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keywordtype">size_t</span> offset_o = 0,</div>
<div class="line"><a id="l00876" name="l00876"></a><span class="lineno">  876</span>                                       <span class="keywordtype">size_t</span> offset_i = 0) {</div>
<div class="line"><a id="l00877" name="l00877"></a><span class="lineno">  877</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l00878" name="l00878"></a><span class="lineno">  878</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> row = idx / n;</div>
<div class="line"><a id="l00879" name="l00879"></a><span class="lineno">  879</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> col = idx % n;</div>
<div class="line"><a id="l00880" name="l00880"></a><span class="lineno">  880</span>        <span class="keywordflow">if</span> (row &lt; M &amp;&amp; col &lt; N) {</div>
<div class="line"><a id="l00881" name="l00881"></a><span class="lineno">  881</span>            atomicAdd(out + (row * N + col + offset_o), in[row * n + col + offset_i]);</div>
<div class="line"><a id="l00882" name="l00882"></a><span class="lineno">  882</span>        }</div>
<div class="line"><a id="l00883" name="l00883"></a><span class="lineno">  883</span>    }</div>
<div class="line"><a id="l00884" name="l00884"></a><span class="lineno">  884</span> </div>
<div class="foldopen" id="foldopen00885" data-start="{" data-end="}">
<div class="line"><a id="l00885" name="l00885"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aa84aa2397f4f5a09a96bef76726e46f0">  885</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aa84aa2397f4f5a09a96bef76726e46f0">TensorCoreGEMM</a>(<span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B, <span class="keywordtype">float</span>* C, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> M,</div>
<div class="line"><a id="l00886" name="l00886"></a><span class="lineno">  886</span>                        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> N, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> K) {</div>
<div class="line"><a id="l00887" name="l00887"></a><span class="lineno">  887</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> m = CEIL(M);</div>
<div class="line"><a id="l00888" name="l00888"></a><span class="lineno">  888</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> k = CEIL(K);</div>
<div class="line"><a id="l00889" name="l00889"></a><span class="lineno">  889</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n = CEIL(N);</div>
<div class="line"><a id="l00890" name="l00890"></a><span class="lineno">  890</span>        half* padded_A;</div>
<div class="line"><a id="l00891" name="l00891"></a><span class="lineno">  891</span>        half* padded_B;</div>
<div class="line"><a id="l00892" name="l00892"></a><span class="lineno">  892</span>        <span class="keywordtype">float</span>* padded_C;</div>
<div class="line"><a id="l00893" name="l00893"></a><span class="lineno">  893</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_A, m * k * <span class="keyword">sizeof</span>(half));</div>
<div class="line"><a id="l00894" name="l00894"></a><span class="lineno">  894</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_B, k * n * <span class="keyword">sizeof</span>(half));</div>
<div class="line"><a id="l00895" name="l00895"></a><span class="lineno">  895</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_C, m * n * <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>));</div>
<div class="line"><a id="l00896" name="l00896"></a><span class="lineno">  896</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a71ad766cb2869d3dd6a3931966e81706">memset</a>(padded_C, 0, m * n * <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>));</div>
<div class="line"><a id="l00897" name="l00897"></a><span class="lineno">  897</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(Padding, dim3((m * k + 256 - 1) / 256), dim3(256), 0, padded_A, A, M, K,</div>
<div class="line"><a id="l00898" name="l00898"></a><span class="lineno">  898</span>                                                m, k, 0, 0);</div>
<div class="line"><a id="l00899" name="l00899"></a><span class="lineno">  899</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(Padding, dim3((k * n + 256 - 1) / 256), dim3(256), 0, padded_B, B, K, N,</div>
<div class="line"><a id="l00900" name="l00900"></a><span class="lineno">  900</span>                                                k, n, 0, 0);</div>
<div class="line"><a id="l00901" name="l00901"></a><span class="lineno">  901</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> tiles = (m * n) &gt;&gt; 8;</div>
<div class="line"><a id="l00902" name="l00902"></a><span class="lineno">  902</span>        dim3 block(256);</div>
<div class="line"><a id="l00903" name="l00903"></a><span class="lineno">  903</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> warpPerBlock = block.x / 32;</div>
<div class="line"><a id="l00904" name="l00904"></a><span class="lineno">  904</span>        dim3 grid((tiles + warpPerBlock - 1) / warpPerBlock);</div>
<div class="line"><a id="l00905" name="l00905"></a><span class="lineno">  905</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(GeneralMatrixMulTensorKernel, grid, block, 0, padded_C, padded_A,</div>
<div class="line"><a id="l00906" name="l00906"></a><span class="lineno">  906</span>                                                padded_B, m, n, k, 0, 0, 0);</div>
<div class="line"><a id="l00907" name="l00907"></a><span class="lineno">  907</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(Cutting, dim3((m * n + 256 - 1) / 256), dim3(256), 0, C, padded_C, M, N,</div>
<div class="line"><a id="l00908" name="l00908"></a><span class="lineno">  908</span>                                                m, n, 0, 0);</div>
<div class="line"><a id="l00909" name="l00909"></a><span class="lineno">  909</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_A);</div>
<div class="line"><a id="l00910" name="l00910"></a><span class="lineno">  910</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_B);</div>
<div class="line"><a id="l00911" name="l00911"></a><span class="lineno">  911</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_C);</div>
<div class="line"><a id="l00912" name="l00912"></a><span class="lineno">  912</span>    }</div>
</div>
<div class="line"><a id="l00913" name="l00913"></a><span class="lineno">  913</span> </div>
<div class="line"><a id="l00914" name="l00914"></a><span class="lineno">  914</span>    <span class="keywordtype">void</span> TensorCoreGEMMParallel(<span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B, <span class="keywordtype">float</span>* C,</div>
<div class="line"><a id="l00915" name="l00915"></a><span class="lineno">  915</span>                                <span class="keyword">const</span> <a class="code hl_class" href="classnz_1_1data_1_1_dimension.html">data::Dimension</a>&amp; A_shape,</div>
<div class="line"><a id="l00916" name="l00916"></a><span class="lineno">  916</span>                                <span class="keyword">const</span> <a class="code hl_class" href="classnz_1_1data_1_1_dimension.html">data::Dimension</a>&amp; B_shape,</div>
<div class="line"><a id="l00917" name="l00917"></a><span class="lineno">  917</span>                                <span class="keyword">const</span> <a class="code hl_class" href="classnz_1_1data_1_1_dimension.html">data::Dimension</a>&amp; C_shape) {</div>
<div class="line"><a id="l00918" name="l00918"></a><span class="lineno">  918</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> M = A_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a7eb3acc882c48e775c418d97f709240f">H</a>();</div>
<div class="line"><a id="l00919" name="l00919"></a><span class="lineno">  919</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> N = B_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a65773c675476dfea3f06b30f21ebbedd">W</a>();</div>
<div class="line"><a id="l00920" name="l00920"></a><span class="lineno">  920</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> K = A_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a65773c675476dfea3f06b30f21ebbedd">W</a>();</div>
<div class="line"><a id="l00921" name="l00921"></a><span class="lineno">  921</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> m = CEIL(M);</div>
<div class="line"><a id="l00922" name="l00922"></a><span class="lineno">  922</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> n = CEIL(N);</div>
<div class="line"><a id="l00923" name="l00923"></a><span class="lineno">  923</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> k = CEIL(K);</div>
<div class="line"><a id="l00924" name="l00924"></a><span class="lineno">  924</span>        <span class="keyword">const</span> <span class="keyword">auto</span> padded_A_shape = <a class="code hl_class" href="classnz_1_1data_1_1_dimension.html">data::Dimension</a>(A_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#acc472e84b4c44f649f34b6fbb0eeacf7">N</a>(), A_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#ae1e87c4a462dd60e02821aa27ffc7e09">C</a>(), m, k);</div>
<div class="line"><a id="l00925" name="l00925"></a><span class="lineno">  925</span>        <span class="keyword">const</span> <span class="keyword">auto</span> padded_B_shape = <a class="code hl_class" href="classnz_1_1data_1_1_dimension.html">data::Dimension</a>(B_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#acc472e84b4c44f649f34b6fbb0eeacf7">N</a>(), B_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#ae1e87c4a462dd60e02821aa27ffc7e09">C</a>(), k, n);</div>
<div class="line"><a id="l00926" name="l00926"></a><span class="lineno">  926</span>        <span class="keyword">const</span> <span class="keyword">auto</span> padded_C_shape = <a class="code hl_class" href="classnz_1_1data_1_1_dimension.html">data::Dimension</a>(C_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#acc472e84b4c44f649f34b6fbb0eeacf7">N</a>(), C_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#ae1e87c4a462dd60e02821aa27ffc7e09">C</a>(), m, n);</div>
<div class="line"><a id="l00927" name="l00927"></a><span class="lineno">  927</span>        half* padded_A;</div>
<div class="line"><a id="l00928" name="l00928"></a><span class="lineno">  928</span>        half* padded_B;</div>
<div class="line"><a id="l00929" name="l00929"></a><span class="lineno">  929</span>        <span class="keywordtype">float</span>* padded_C;</div>
<div class="line"><a id="l00930" name="l00930"></a><span class="lineno">  930</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_A, padded_A_shape.size() * <span class="keyword">sizeof</span>(half));</div>
<div class="line"><a id="l00931" name="l00931"></a><span class="lineno">  931</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_B, padded_B_shape.size() * <span class="keyword">sizeof</span>(half));</div>
<div class="line"><a id="l00932" name="l00932"></a><span class="lineno">  932</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_C, padded_C_shape.size() * <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>));</div>
<div class="line"><a id="l00933" name="l00933"></a><span class="lineno">  933</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a71ad766cb2869d3dd6a3931966e81706">memset</a>(padded_C, 0, padded_C_shape.size() * <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>));</div>
<div class="line"><a id="l00934" name="l00934"></a><span class="lineno">  934</span>        std::vector&lt;size_t&gt; offset1;</div>
<div class="line"><a id="l00935" name="l00935"></a><span class="lineno">  935</span>        std::vector&lt;size_t&gt; offset2;</div>
<div class="line"><a id="l00936" name="l00936"></a><span class="lineno">  936</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; A_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#acc472e84b4c44f649f34b6fbb0eeacf7">N</a>(); i++) {</div>
<div class="line"><a id="l00937" name="l00937"></a><span class="lineno">  937</span>            <span class="keywordflow">for</span> (<span class="keyword">auto</span> j = 0; j &lt; A_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#ae1e87c4a462dd60e02821aa27ffc7e09">C</a>(); j++) {</div>
<div class="line"><a id="l00938" name="l00938"></a><span class="lineno">  938</span>                offset1.push_back(i * A_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a4831fea5aaf7dbad3578d3fa8e55aef1">getStride</a>(0) + j * A_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a4831fea5aaf7dbad3578d3fa8e55aef1">getStride</a>(1));</div>
<div class="line"><a id="l00939" name="l00939"></a><span class="lineno">  939</span>                offset2.push_back(i * padded_A_shape.getStride(0) + j * padded_A_shape.getStride(1));</div>
<div class="line"><a id="l00940" name="l00940"></a><span class="lineno">  940</span>            }</div>
<div class="line"><a id="l00941" name="l00941"></a><span class="lineno">  941</span>        }</div>
<div class="line"><a id="l00942" name="l00942"></a><span class="lineno">  942</span>        std::vector&lt;cudaStream_t&gt; streams;</div>
<div class="line"><a id="l00943" name="l00943"></a><span class="lineno">  943</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset1.size(); i++) {</div>
<div class="line"><a id="l00944" name="l00944"></a><span class="lineno">  944</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l00945" name="l00945"></a><span class="lineno">  945</span>            streams.push_back(stream);</div>
<div class="line"><a id="l00946" name="l00946"></a><span class="lineno">  946</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(A, stream);</div>
<div class="line"><a id="l00947" name="l00947"></a><span class="lineno">  947</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_A, stream);</div>
<div class="line"><a id="l00948" name="l00948"></a><span class="lineno">  948</span>            Padding&lt;&lt;&lt;dim3((m * k + 256 - 1) / 256), dim3(256), 0, stream&gt;&gt;&gt;(</div>
<div class="line"><a id="l00949" name="l00949"></a><span class="lineno">  949</span>                padded_A, A, M, K, m, k, offset2[i], offset1[i]);</div>
<div class="line"><a id="l00950" name="l00950"></a><span class="lineno">  950</span>        }</div>
<div class="line"><a id="l00951" name="l00951"></a><span class="lineno">  951</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l00952" name="l00952"></a><span class="lineno">  952</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(padded_A, stream);</div>
<div class="line"><a id="l00953" name="l00953"></a><span class="lineno">  953</span>        }</div>
<div class="line"><a id="l00954" name="l00954"></a><span class="lineno">  954</span>        offset1.clear();</div>
<div class="line"><a id="l00955" name="l00955"></a><span class="lineno">  955</span>        offset2.clear();</div>
<div class="line"><a id="l00956" name="l00956"></a><span class="lineno">  956</span>        streams.clear();</div>
<div class="line"><a id="l00957" name="l00957"></a><span class="lineno">  957</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; B_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#acc472e84b4c44f649f34b6fbb0eeacf7">N</a>(); i++) {</div>
<div class="line"><a id="l00958" name="l00958"></a><span class="lineno">  958</span>            <span class="keywordflow">for</span> (<span class="keyword">auto</span> j = 0; j &lt; B_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#ae1e87c4a462dd60e02821aa27ffc7e09">C</a>(); j++) {</div>
<div class="line"><a id="l00959" name="l00959"></a><span class="lineno">  959</span>                offset1.push_back(i * B_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a4831fea5aaf7dbad3578d3fa8e55aef1">getStride</a>(0) + j * B_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a4831fea5aaf7dbad3578d3fa8e55aef1">getStride</a>(1));</div>
<div class="line"><a id="l00960" name="l00960"></a><span class="lineno">  960</span>                offset2.push_back(i * padded_B_shape.getStride(0) + j * padded_B_shape.getStride(1));</div>
<div class="line"><a id="l00961" name="l00961"></a><span class="lineno">  961</span>            }</div>
<div class="line"><a id="l00962" name="l00962"></a><span class="lineno">  962</span>        }</div>
<div class="line"><a id="l00963" name="l00963"></a><span class="lineno">  963</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset1.size(); i++) {</div>
<div class="line"><a id="l00964" name="l00964"></a><span class="lineno">  964</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l00965" name="l00965"></a><span class="lineno">  965</span>            streams.push_back(stream);</div>
<div class="line"><a id="l00966" name="l00966"></a><span class="lineno">  966</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(B, stream);</div>
<div class="line"><a id="l00967" name="l00967"></a><span class="lineno">  967</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_B, stream);</div>
<div class="line"><a id="l00968" name="l00968"></a><span class="lineno">  968</span>            Padding&lt;&lt;&lt;dim3((k * n + 256 - 1) / 256), dim3(256), 0, stream&gt;&gt;&gt;(</div>
<div class="line"><a id="l00969" name="l00969"></a><span class="lineno">  969</span>                padded_B, B, K, N, k, n, offset2[i], offset1[i]);</div>
<div class="line"><a id="l00970" name="l00970"></a><span class="lineno">  970</span>        }</div>
<div class="line"><a id="l00971" name="l00971"></a><span class="lineno">  971</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l00972" name="l00972"></a><span class="lineno">  972</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(padded_B, stream);</div>
<div class="line"><a id="l00973" name="l00973"></a><span class="lineno">  973</span>        }</div>
<div class="line"><a id="l00974" name="l00974"></a><span class="lineno">  974</span>        offset1.clear();</div>
<div class="line"><a id="l00975" name="l00975"></a><span class="lineno">  975</span>        offset2.clear();</div>
<div class="line"><a id="l00976" name="l00976"></a><span class="lineno">  976</span>        streams.clear();</div>
<div class="line"><a id="l00977" name="l00977"></a><span class="lineno">  977</span>        std::vector&lt;size_t&gt; offset3;</div>
<div class="line"><a id="l00978" name="l00978"></a><span class="lineno">  978</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; padded_C_shape[0]; i++) {</div>
<div class="line"><a id="l00979" name="l00979"></a><span class="lineno">  979</span>            <span class="keywordflow">for</span> (<span class="keyword">auto</span> j = 0; j &lt; padded_C_shape[1]; j++) {</div>
<div class="line"><a id="l00980" name="l00980"></a><span class="lineno">  980</span>                offset3.push_back(i * padded_C_shape.getStride(0) + j * padded_C_shape.getStride(1));</div>
<div class="line"><a id="l00981" name="l00981"></a><span class="lineno">  981</span>                offset1.push_back(i * (padded_A_shape[0] &gt; 1 ? padded_A_shape.getStride(0) : 0) +</div>
<div class="line"><a id="l00982" name="l00982"></a><span class="lineno">  982</span>                    j * (padded_A_shape[1] &gt; 1 ? padded_A_shape.getStride(1) : 0));</div>
<div class="line"><a id="l00983" name="l00983"></a><span class="lineno">  983</span>                offset2.push_back(i * (padded_B_shape[0] &gt; 1 ? padded_B_shape.getStride(0) : 0) +</div>
<div class="line"><a id="l00984" name="l00984"></a><span class="lineno">  984</span>                    j * (padded_B_shape[1] &gt; 1 ? padded_B_shape.getStride(1) : 0));</div>
<div class="line"><a id="l00985" name="l00985"></a><span class="lineno">  985</span>            }</div>
<div class="line"><a id="l00986" name="l00986"></a><span class="lineno">  986</span>        }</div>
<div class="line"><a id="l00987" name="l00987"></a><span class="lineno">  987</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> tiles = (m * n) &gt;&gt; 8;</div>
<div class="line"><a id="l00988" name="l00988"></a><span class="lineno">  988</span>        dim3 block(256);</div>
<div class="line"><a id="l00989" name="l00989"></a><span class="lineno">  989</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> warpPerBlock = block.x / 32;</div>
<div class="line"><a id="l00990" name="l00990"></a><span class="lineno">  990</span>        dim3 grid((tiles + warpPerBlock - 1) / warpPerBlock);</div>
<div class="line"><a id="l00991" name="l00991"></a><span class="lineno">  991</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset3.size(); i++) {</div>
<div class="line"><a id="l00992" name="l00992"></a><span class="lineno">  992</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l00993" name="l00993"></a><span class="lineno">  993</span>            streams.push_back(stream);</div>
<div class="line"><a id="l00994" name="l00994"></a><span class="lineno">  994</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_C, stream);</div>
<div class="line"><a id="l00995" name="l00995"></a><span class="lineno">  995</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_A, stream);</div>
<div class="line"><a id="l00996" name="l00996"></a><span class="lineno">  996</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_B, stream);</div>
<div class="line"><a id="l00997" name="l00997"></a><span class="lineno">  997</span>            GeneralMatrixMulTensorKernel&lt;&lt;&lt;grid, block, 0, stream&gt;&gt;&gt;(padded_C, padded_A, padded_B, m, n, k, offset3[i],</div>
<div class="line"><a id="l00998" name="l00998"></a><span class="lineno">  998</span>                                                                     offset1[i], offset2[i]);</div>
<div class="line"><a id="l00999" name="l00999"></a><span class="lineno">  999</span>        }</div>
<div class="line"><a id="l01000" name="l01000"></a><span class="lineno"> 1000</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l01001" name="l01001"></a><span class="lineno"> 1001</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(padded_C, stream);</div>
<div class="line"><a id="l01002" name="l01002"></a><span class="lineno"> 1002</span>        }</div>
<div class="line"><a id="l01003" name="l01003"></a><span class="lineno"> 1003</span>        offset1.clear();</div>
<div class="line"><a id="l01004" name="l01004"></a><span class="lineno"> 1004</span>        offset2.clear();</div>
<div class="line"><a id="l01005" name="l01005"></a><span class="lineno"> 1005</span>        streams.clear();</div>
<div class="line"><a id="l01006" name="l01006"></a><span class="lineno"> 1006</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; C_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#acc472e84b4c44f649f34b6fbb0eeacf7">N</a>(); i++) {</div>
<div class="line"><a id="l01007" name="l01007"></a><span class="lineno"> 1007</span>            <span class="keywordflow">for</span> (<span class="keyword">auto</span> j = 0; j &lt; C_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#ae1e87c4a462dd60e02821aa27ffc7e09">C</a>(); j++) {</div>
<div class="line"><a id="l01008" name="l01008"></a><span class="lineno"> 1008</span>                offset1.push_back(i * C_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a4831fea5aaf7dbad3578d3fa8e55aef1">getStride</a>(0) + j * C_shape.<a class="code hl_function" href="classnz_1_1data_1_1_dimension.html#a4831fea5aaf7dbad3578d3fa8e55aef1">getStride</a>(1));</div>
<div class="line"><a id="l01009" name="l01009"></a><span class="lineno"> 1009</span>                offset2.push_back(i * padded_C_shape.getStride(0) + j * padded_C_shape.getStride(1));</div>
<div class="line"><a id="l01010" name="l01010"></a><span class="lineno"> 1010</span>            }</div>
<div class="line"><a id="l01011" name="l01011"></a><span class="lineno"> 1011</span>        }</div>
<div class="line"><a id="l01012" name="l01012"></a><span class="lineno"> 1012</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset1.size(); i++) {</div>
<div class="line"><a id="l01013" name="l01013"></a><span class="lineno"> 1013</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l01014" name="l01014"></a><span class="lineno"> 1014</span>            streams.push_back(stream);</div>
<div class="line"><a id="l01015" name="l01015"></a><span class="lineno"> 1015</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(C, stream);</div>
<div class="line"><a id="l01016" name="l01016"></a><span class="lineno"> 1016</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_C, stream);</div>
<div class="line"><a id="l01017" name="l01017"></a><span class="lineno"> 1017</span>            Cutting&lt;&lt;&lt;dim3((n * m + 256 - 1) / 256), dim3(256), 0, stream&gt;&gt;&gt;(</div>
<div class="line"><a id="l01018" name="l01018"></a><span class="lineno"> 1018</span>                C, padded_C, M, N, m, n, offset1[i], offset2[i]);</div>
<div class="line"><a id="l01019" name="l01019"></a><span class="lineno"> 1019</span>        }</div>
<div class="line"><a id="l01020" name="l01020"></a><span class="lineno"> 1020</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l01021" name="l01021"></a><span class="lineno"> 1021</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(C, stream);</div>
<div class="line"><a id="l01022" name="l01022"></a><span class="lineno"> 1022</span>        }</div>
<div class="line"><a id="l01023" name="l01023"></a><span class="lineno"> 1023</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_A);</div>
<div class="line"><a id="l01024" name="l01024"></a><span class="lineno"> 1024</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_B);</div>
<div class="line"><a id="l01025" name="l01025"></a><span class="lineno"> 1025</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_C);</div>
<div class="line"><a id="l01026" name="l01026"></a><span class="lineno"> 1026</span>    }</div>
<div class="line"><a id="l01027" name="l01027"></a><span class="lineno"> 1027</span> </div>
<div class="line"><a id="l01028" name="l01028"></a><span class="lineno"> 1028</span>    <span class="keywordtype">void</span> GEMMBackwardParallel(<span class="keywordtype">float</span>* A, <span class="keywordtype">float</span>* B, <span class="keywordtype">float</span>* C,</div>
<div class="line"><a id="l01029" name="l01029"></a><span class="lineno"> 1029</span>                              <span class="keyword">const</span> data::Dimension&amp; A_shape,</div>
<div class="line"><a id="l01030" name="l01030"></a><span class="lineno"> 1030</span>                              <span class="keyword">const</span> data::Dimension&amp; B_shape,</div>
<div class="line"><a id="l01031" name="l01031"></a><span class="lineno"> 1031</span>                              <span class="keyword">const</span> data::Dimension&amp; C_shape) {</div>
<div class="line"><a id="l01032" name="l01032"></a><span class="lineno"> 1032</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> M = A_shape.H();</div>
<div class="line"><a id="l01033" name="l01033"></a><span class="lineno"> 1033</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> N = B_shape.W();</div>
<div class="line"><a id="l01034" name="l01034"></a><span class="lineno"> 1034</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> K = A_shape.W();</div>
<div class="line"><a id="l01035" name="l01035"></a><span class="lineno"> 1035</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> m = CEIL(M);</div>
<div class="line"><a id="l01036" name="l01036"></a><span class="lineno"> 1036</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> n = CEIL(N);</div>
<div class="line"><a id="l01037" name="l01037"></a><span class="lineno"> 1037</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> k = CEIL(K);</div>
<div class="line"><a id="l01038" name="l01038"></a><span class="lineno"> 1038</span>        <span class="keyword">const</span> <span class="keyword">auto</span> padded_A_shape = data::Dimension(A_shape.N(), A_shape.C(), m, k);</div>
<div class="line"><a id="l01039" name="l01039"></a><span class="lineno"> 1039</span>        <span class="keyword">const</span> <span class="keyword">auto</span> padded_B_shape = data::Dimension(B_shape.N(), B_shape.C(), k, n);</div>
<div class="line"><a id="l01040" name="l01040"></a><span class="lineno"> 1040</span>        <span class="keyword">const</span> <span class="keyword">auto</span> padded_C_shape = data::Dimension(std::max(A_shape.N(), B_shape.N()),</div>
<div class="line"><a id="l01041" name="l01041"></a><span class="lineno"> 1041</span>                                                    std::max(A_shape.C(), B_shape.C()), m, n);</div>
<div class="line"><a id="l01042" name="l01042"></a><span class="lineno"> 1042</span>        half* padded_A;</div>
<div class="line"><a id="l01043" name="l01043"></a><span class="lineno"> 1043</span>        half* padded_B;</div>
<div class="line"><a id="l01044" name="l01044"></a><span class="lineno"> 1044</span>        <span class="keywordtype">float</span>* padded_C;</div>
<div class="line"><a id="l01045" name="l01045"></a><span class="lineno"> 1045</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_A, padded_A_shape.size() * <span class="keyword">sizeof</span>(half));</div>
<div class="line"><a id="l01046" name="l01046"></a><span class="lineno"> 1046</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_B, padded_B_shape.size() * <span class="keyword">sizeof</span>(half));</div>
<div class="line"><a id="l01047" name="l01047"></a><span class="lineno"> 1047</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">malloc</a>(&amp;padded_C, padded_C_shape.size() * <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>));</div>
<div class="line"><a id="l01048" name="l01048"></a><span class="lineno"> 1048</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a71ad766cb2869d3dd6a3931966e81706">memset</a>(padded_C, 0, padded_C_shape.size() * <span class="keyword">sizeof</span>(<span class="keywordtype">float</span>));</div>
<div class="line"><a id="l01049" name="l01049"></a><span class="lineno"> 1049</span>        std::vector&lt;size_t&gt; offset1;</div>
<div class="line"><a id="l01050" name="l01050"></a><span class="lineno"> 1050</span>        std::vector&lt;size_t&gt; offset2;</div>
<div class="line"><a id="l01051" name="l01051"></a><span class="lineno"> 1051</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; A_shape.N(); i++) {</div>
<div class="line"><a id="l01052" name="l01052"></a><span class="lineno"> 1052</span>            <span class="keywordflow">for</span> (<span class="keyword">auto</span> j = 0; j &lt; A_shape.C(); j++) {</div>
<div class="line"><a id="l01053" name="l01053"></a><span class="lineno"> 1053</span>                offset1.push_back(i * A_shape.getStride(0) + j * A_shape.getStride(1));</div>
<div class="line"><a id="l01054" name="l01054"></a><span class="lineno"> 1054</span>                offset2.push_back(i * padded_A_shape.getStride(0) + j * padded_A_shape.getStride(1));</div>
<div class="line"><a id="l01055" name="l01055"></a><span class="lineno"> 1055</span>            }</div>
<div class="line"><a id="l01056" name="l01056"></a><span class="lineno"> 1056</span>        }</div>
<div class="line"><a id="l01057" name="l01057"></a><span class="lineno"> 1057</span>        std::vector&lt;cudaStream_t&gt; streams;</div>
<div class="line"><a id="l01058" name="l01058"></a><span class="lineno"> 1058</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset1.size(); i++) {</div>
<div class="line"><a id="l01059" name="l01059"></a><span class="lineno"> 1059</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l01060" name="l01060"></a><span class="lineno"> 1060</span>            streams.push_back(stream);</div>
<div class="line"><a id="l01061" name="l01061"></a><span class="lineno"> 1061</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(A, stream);</div>
<div class="line"><a id="l01062" name="l01062"></a><span class="lineno"> 1062</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_A, stream);</div>
<div class="line"><a id="l01063" name="l01063"></a><span class="lineno"> 1063</span>            Padding&lt;&lt;&lt;dim3((m * k + 256 - 1) / 256), dim3(256), 0, stream&gt;&gt;&gt;(</div>
<div class="line"><a id="l01064" name="l01064"></a><span class="lineno"> 1064</span>                padded_A, A, M, K, m, k, offset2[i], offset1[i]);</div>
<div class="line"><a id="l01065" name="l01065"></a><span class="lineno"> 1065</span>        }</div>
<div class="line"><a id="l01066" name="l01066"></a><span class="lineno"> 1066</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l01067" name="l01067"></a><span class="lineno"> 1067</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(padded_A, stream);</div>
<div class="line"><a id="l01068" name="l01068"></a><span class="lineno"> 1068</span>        }</div>
<div class="line"><a id="l01069" name="l01069"></a><span class="lineno"> 1069</span>        offset1.clear();</div>
<div class="line"><a id="l01070" name="l01070"></a><span class="lineno"> 1070</span>        offset2.clear();</div>
<div class="line"><a id="l01071" name="l01071"></a><span class="lineno"> 1071</span>        streams.clear();</div>
<div class="line"><a id="l01072" name="l01072"></a><span class="lineno"> 1072</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; B_shape.N(); i++) {</div>
<div class="line"><a id="l01073" name="l01073"></a><span class="lineno"> 1073</span>            <span class="keywordflow">for</span> (<span class="keyword">auto</span> j = 0; j &lt; B_shape.C(); j++) {</div>
<div class="line"><a id="l01074" name="l01074"></a><span class="lineno"> 1074</span>                offset1.push_back(i * B_shape.getStride(0) + j * B_shape.getStride(1));</div>
<div class="line"><a id="l01075" name="l01075"></a><span class="lineno"> 1075</span>                offset2.push_back(i * padded_B_shape.getStride(0) + j * padded_B_shape.getStride(1));</div>
<div class="line"><a id="l01076" name="l01076"></a><span class="lineno"> 1076</span>            }</div>
<div class="line"><a id="l01077" name="l01077"></a><span class="lineno"> 1077</span>        }</div>
<div class="line"><a id="l01078" name="l01078"></a><span class="lineno"> 1078</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset1.size(); i++) {</div>
<div class="line"><a id="l01079" name="l01079"></a><span class="lineno"> 1079</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l01080" name="l01080"></a><span class="lineno"> 1080</span>            streams.push_back(stream);</div>
<div class="line"><a id="l01081" name="l01081"></a><span class="lineno"> 1081</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(B, stream);</div>
<div class="line"><a id="l01082" name="l01082"></a><span class="lineno"> 1082</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_B, stream);</div>
<div class="line"><a id="l01083" name="l01083"></a><span class="lineno"> 1083</span>            Padding&lt;&lt;&lt;dim3((k * n + 256 - 1) / 256), dim3(256), 0, stream&gt;&gt;&gt;(</div>
<div class="line"><a id="l01084" name="l01084"></a><span class="lineno"> 1084</span>                padded_B, B, K, N, k, n, offset2[i], offset1[i]);</div>
<div class="line"><a id="l01085" name="l01085"></a><span class="lineno"> 1085</span>        }</div>
<div class="line"><a id="l01086" name="l01086"></a><span class="lineno"> 1086</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l01087" name="l01087"></a><span class="lineno"> 1087</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(padded_B, stream);</div>
<div class="line"><a id="l01088" name="l01088"></a><span class="lineno"> 1088</span>        }</div>
<div class="line"><a id="l01089" name="l01089"></a><span class="lineno"> 1089</span>        offset1.clear();</div>
<div class="line"><a id="l01090" name="l01090"></a><span class="lineno"> 1090</span>        offset2.clear();</div>
<div class="line"><a id="l01091" name="l01091"></a><span class="lineno"> 1091</span>        streams.clear();</div>
<div class="line"><a id="l01092" name="l01092"></a><span class="lineno"> 1092</span>        std::vector&lt;size_t&gt; offset3;</div>
<div class="line"><a id="l01093" name="l01093"></a><span class="lineno"> 1093</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; padded_C_shape[0]; i++) {</div>
<div class="line"><a id="l01094" name="l01094"></a><span class="lineno"> 1094</span>            <span class="keywordflow">for</span> (<span class="keyword">auto</span> j = 0; j &lt; padded_C_shape[1]; j++) {</div>
<div class="line"><a id="l01095" name="l01095"></a><span class="lineno"> 1095</span>                offset3.push_back(i * padded_C_shape.getStride(0) + j * padded_C_shape.getStride(1));</div>
<div class="line"><a id="l01096" name="l01096"></a><span class="lineno"> 1096</span>                offset1.push_back(i * (padded_A_shape[0] &gt; 1 ? padded_A_shape.getStride(0) : 0) +</div>
<div class="line"><a id="l01097" name="l01097"></a><span class="lineno"> 1097</span>                    j * (padded_A_shape[1] &gt; 1 ? padded_A_shape.getStride(1) : 0));</div>
<div class="line"><a id="l01098" name="l01098"></a><span class="lineno"> 1098</span>                offset2.push_back(i * (padded_B_shape[0] &gt; 1 ? padded_B_shape.getStride(0) : 0) +</div>
<div class="line"><a id="l01099" name="l01099"></a><span class="lineno"> 1099</span>                    j * (padded_B_shape[1] &gt; 1 ? padded_B_shape.getStride(1) : 0));</div>
<div class="line"><a id="l01100" name="l01100"></a><span class="lineno"> 1100</span>            }</div>
<div class="line"><a id="l01101" name="l01101"></a><span class="lineno"> 1101</span>        }</div>
<div class="line"><a id="l01102" name="l01102"></a><span class="lineno"> 1102</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> tiles = (m * n) &gt;&gt; 8;</div>
<div class="line"><a id="l01103" name="l01103"></a><span class="lineno"> 1103</span>        dim3 block(256);</div>
<div class="line"><a id="l01104" name="l01104"></a><span class="lineno"> 1104</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> warpPerBlock = block.x / 32;</div>
<div class="line"><a id="l01105" name="l01105"></a><span class="lineno"> 1105</span>        dim3 grid((tiles + warpPerBlock - 1) / warpPerBlock);</div>
<div class="line"><a id="l01106" name="l01106"></a><span class="lineno"> 1106</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset3.size(); i++) {</div>
<div class="line"><a id="l01107" name="l01107"></a><span class="lineno"> 1107</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l01108" name="l01108"></a><span class="lineno"> 1108</span>            streams.push_back(stream);</div>
<div class="line"><a id="l01109" name="l01109"></a><span class="lineno"> 1109</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_C, stream);</div>
<div class="line"><a id="l01110" name="l01110"></a><span class="lineno"> 1110</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_A, stream);</div>
<div class="line"><a id="l01111" name="l01111"></a><span class="lineno"> 1111</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_B, stream);</div>
<div class="line"><a id="l01112" name="l01112"></a><span class="lineno"> 1112</span>            GeneralMatrixMulTensorKernel&lt;&lt;&lt;grid, block, 0, stream&gt;&gt;&gt;(padded_C, padded_A, padded_B, m, n, k, offset3[i],</div>
<div class="line"><a id="l01113" name="l01113"></a><span class="lineno"> 1113</span>                                                                     offset1[i], offset2[i]);</div>
<div class="line"><a id="l01114" name="l01114"></a><span class="lineno"> 1114</span>        }</div>
<div class="line"><a id="l01115" name="l01115"></a><span class="lineno"> 1115</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l01116" name="l01116"></a><span class="lineno"> 1116</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(padded_C, stream);</div>
<div class="line"><a id="l01117" name="l01117"></a><span class="lineno"> 1117</span>        }</div>
<div class="line"><a id="l01118" name="l01118"></a><span class="lineno"> 1118</span>        offset1.clear();</div>
<div class="line"><a id="l01119" name="l01119"></a><span class="lineno"> 1119</span>        offset2.clear();</div>
<div class="line"><a id="l01120" name="l01120"></a><span class="lineno"> 1120</span>        streams.clear();</div>
<div class="line"><a id="l01121" name="l01121"></a><span class="lineno"> 1121</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; padded_C_shape.N(); i++) {</div>
<div class="line"><a id="l01122" name="l01122"></a><span class="lineno"> 1122</span>            <span class="keywordflow">for</span> (<span class="keyword">auto</span> j = 0; j &lt; padded_C_shape.C(); j++) {</div>
<div class="line"><a id="l01123" name="l01123"></a><span class="lineno"> 1123</span>                offset1.push_back(</div>
<div class="line"><a id="l01124" name="l01124"></a><span class="lineno"> 1124</span>                    i * (C_shape.N() &gt; 1 ? C_shape.getStride(0) : 0) + j * (C_shape.C() &gt; 1</div>
<div class="line"><a id="l01125" name="l01125"></a><span class="lineno"> 1125</span>                                                                                ? C_shape.getStride(1)</div>
<div class="line"><a id="l01126" name="l01126"></a><span class="lineno"> 1126</span>                                                                                : 0));</div>
<div class="line"><a id="l01127" name="l01127"></a><span class="lineno"> 1127</span>                offset2.push_back(i * padded_C_shape.getStride(0) + j * padded_C_shape.getStride(1));</div>
<div class="line"><a id="l01128" name="l01128"></a><span class="lineno"> 1128</span>            }</div>
<div class="line"><a id="l01129" name="l01129"></a><span class="lineno"> 1129</span>        }</div>
<div class="line"><a id="l01130" name="l01130"></a><span class="lineno"> 1130</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset1.size(); i++) {</div>
<div class="line"><a id="l01131" name="l01131"></a><span class="lineno"> 1131</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l01132" name="l01132"></a><span class="lineno"> 1132</span>            streams.push_back(stream);</div>
<div class="line"><a id="l01133" name="l01133"></a><span class="lineno"> 1133</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(C, stream);</div>
<div class="line"><a id="l01134" name="l01134"></a><span class="lineno"> 1134</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(padded_C, stream);</div>
<div class="line"><a id="l01135" name="l01135"></a><span class="lineno"> 1135</span>            CuttingAndCompress&lt;&lt;&lt;dim3((n * m + 256 - 1) / 256), dim3(256), 0, stream&gt;&gt;&gt;(</div>
<div class="line"><a id="l01136" name="l01136"></a><span class="lineno"> 1136</span>                C, padded_C, M, N, m, n, offset1[i], offset2[i]);</div>
<div class="line"><a id="l01137" name="l01137"></a><span class="lineno"> 1137</span>        }</div>
<div class="line"><a id="l01138" name="l01138"></a><span class="lineno"> 1138</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l01139" name="l01139"></a><span class="lineno"> 1139</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(C, stream);</div>
<div class="line"><a id="l01140" name="l01140"></a><span class="lineno"> 1140</span>        }</div>
<div class="line"><a id="l01141" name="l01141"></a><span class="lineno"> 1141</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_A);</div>
<div class="line"><a id="l01142" name="l01142"></a><span class="lineno"> 1142</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_B);</div>
<div class="line"><a id="l01143" name="l01143"></a><span class="lineno"> 1143</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">freeAsync</a>(padded_C);</div>
<div class="line"><a id="l01144" name="l01144"></a><span class="lineno"> 1144</span>    }</div>
<div class="line"><a id="l01145" name="l01145"></a><span class="lineno"> 1145</span> </div>
<div class="line"><a id="l01146" name="l01146"></a><span class="lineno"> 1146</span>    __global__ <span class="keywordtype">void</span> FillKernel(<span class="keywordtype">float</span>* data, <span class="keyword">const</span> <span class="keywordtype">float</span> value, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l01147" name="l01147"></a><span class="lineno"> 1147</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01148" name="l01148"></a><span class="lineno"> 1148</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l01149" name="l01149"></a><span class="lineno"> 1149</span>            data[idx + offset] = value;</div>
<div class="line"><a id="l01150" name="l01150"></a><span class="lineno"> 1150</span>        }</div>
<div class="line"><a id="l01151" name="l01151"></a><span class="lineno"> 1151</span>    }</div>
<div class="line"><a id="l01152" name="l01152"></a><span class="lineno"> 1152</span> </div>
<div class="foldopen" id="foldopen01153" data-start="{" data-end="}">
<div class="line"><a id="l01153" name="l01153"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#ad136c8a6560a5305984ce0a31bea71bf"> 1153</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#ad136c8a6560a5305984ce0a31bea71bf">Fill</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* data, <span class="keyword">const</span> <span class="keywordtype">float</span> value, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l01154" name="l01154"></a><span class="lineno"> 1154</span>              <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l01155" name="l01155"></a><span class="lineno"> 1155</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(FillKernel, gridDim, blockDim, 0, data, value, n, offset);</div>
<div class="line"><a id="l01156" name="l01156"></a><span class="lineno"> 1156</span>    }</div>
</div>
<div class="line"><a id="l01157" name="l01157"></a><span class="lineno"> 1157</span> </div>
<div class="line"><a id="l01158" name="l01158"></a><span class="lineno"> 1158</span>    __global__ <span class="keywordtype">void</span> HadamardProductKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in1, <span class="keyword">const</span> <span class="keywordtype">float</span>* in2, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l01159" name="l01159"></a><span class="lineno"> 1159</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01160" name="l01160"></a><span class="lineno"> 1160</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l01161" name="l01161"></a><span class="lineno"> 1161</span>            out[idx] = in1[idx] * in2[idx];</div>
<div class="line"><a id="l01162" name="l01162"></a><span class="lineno"> 1162</span>        }</div>
<div class="line"><a id="l01163" name="l01163"></a><span class="lineno"> 1163</span>    }</div>
<div class="line"><a id="l01164" name="l01164"></a><span class="lineno"> 1164</span> </div>
<div class="foldopen" id="foldopen01165" data-start="{" data-end="}">
<div class="line"><a id="l01165" name="l01165"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a8ec4524fdefd3d771c72e77e94281c88"> 1165</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a8ec4524fdefd3d771c72e77e94281c88">HadamardProduct</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in1, <span class="keywordtype">float</span>* in2,</div>
<div class="line"><a id="l01166" name="l01166"></a><span class="lineno"> 1166</span>                         <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n) {</div>
<div class="line"><a id="l01167" name="l01167"></a><span class="lineno"> 1167</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(HadamardProductKernel, gridDim, blockDim, 0, out, in1, in2, n);</div>
<div class="line"><a id="l01168" name="l01168"></a><span class="lineno"> 1168</span>    }</div>
</div>
<div class="line"><a id="l01169" name="l01169"></a><span class="lineno"> 1169</span> </div>
<div class="line"><a id="l01170" name="l01170"></a><span class="lineno"> 1170</span>    __global__ <span class="keywordtype">void</span> ElementwiseDivideKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in1, <span class="keyword">const</span> <span class="keywordtype">float</span>* in2,</div>
<div class="line"><a id="l01171" name="l01171"></a><span class="lineno"> 1171</span>                                            <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n,</div>
<div class="line"><a id="l01172" name="l01172"></a><span class="lineno"> 1172</span>                                            <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_o,</div>
<div class="line"><a id="l01173" name="l01173"></a><span class="lineno"> 1173</span>                                            <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_1,</div>
<div class="line"><a id="l01174" name="l01174"></a><span class="lineno"> 1174</span>                                            <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_2) {</div>
<div class="line"><a id="l01175" name="l01175"></a><span class="lineno"> 1175</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01176" name="l01176"></a><span class="lineno"> 1176</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l01177" name="l01177"></a><span class="lineno"> 1177</span>            out[idx + offset_o] = in1[idx + offset_1] / in2[idx + offset_2];</div>
<div class="line"><a id="l01178" name="l01178"></a><span class="lineno"> 1178</span>        }</div>
<div class="line"><a id="l01179" name="l01179"></a><span class="lineno"> 1179</span>    }</div>
<div class="line"><a id="l01180" name="l01180"></a><span class="lineno"> 1180</span> </div>
<div class="foldopen" id="foldopen01181" data-start="{" data-end="}">
<div class="line"><a id="l01181" name="l01181"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#aa61cded4977bb2dc3720f7057cc2fb47"> 1181</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aa61cded4977bb2dc3720f7057cc2fb47">ElementwiseDivide</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in1, <span class="keywordtype">float</span>* in2,</div>
<div class="line"><a id="l01182" name="l01182"></a><span class="lineno"> 1182</span>                           <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_o,</div>
<div class="line"><a id="l01183" name="l01183"></a><span class="lineno"> 1183</span>                           <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_1,</div>
<div class="line"><a id="l01184" name="l01184"></a><span class="lineno"> 1184</span>                           <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_2) {</div>
<div class="line"><a id="l01185" name="l01185"></a><span class="lineno"> 1185</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(ElementwiseDivideKernel, gridDim, blockDim, 0, out, in1, in2, n,</div>
<div class="line"><a id="l01186" name="l01186"></a><span class="lineno"> 1186</span>                                                offset_o, offset_1, offset_2);</div>
<div class="line"><a id="l01187" name="l01187"></a><span class="lineno"> 1187</span>    }</div>
</div>
<div class="line"><a id="l01188" name="l01188"></a><span class="lineno"> 1188</span> </div>
<div class="line"><a id="l01189" name="l01189"></a><span class="lineno"> 1189</span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#aa61cded4977bb2dc3720f7057cc2fb47">ElementwiseDivide</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in1, <span class="keywordtype">float</span>* in2,</div>
<div class="line"><a id="l01190" name="l01190"></a><span class="lineno"> 1190</span>                           <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_o,</div>
<div class="line"><a id="l01191" name="l01191"></a><span class="lineno"> 1191</span>                           <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_1,</div>
<div class="line"><a id="l01192" name="l01192"></a><span class="lineno"> 1192</span>                           <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_2) {</div>
<div class="line"><a id="l01193" name="l01193"></a><span class="lineno"> 1193</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitParallel(ElementwiseDivideKernel, gridDim, blockDim, 0, out, in1, in2,</div>
<div class="line"><a id="l01194" name="l01194"></a><span class="lineno"> 1194</span>                                                        offset_o, offset_1, offset_2, n);</div>
<div class="line"><a id="l01195" name="l01195"></a><span class="lineno"> 1195</span>    }</div>
<div class="line"><a id="l01196" name="l01196"></a><span class="lineno"> 1196</span> </div>
<div class="line"><a id="l01197" name="l01197"></a><span class="lineno"> 1197</span>    __global__ <span class="keywordtype">void</span> SummationKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l01198" name="l01198"></a><span class="lineno"> 1198</span>        <span class="keyword">extern</span> __shared__ <span class="keywordtype">float</span> sdata[];</div>
<div class="line"><a id="l01199" name="l01199"></a><span class="lineno"> 1199</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> tid = threadIdx.x;</div>
<div class="line"><a id="l01200" name="l01200"></a><span class="lineno"> 1200</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> idx = blockDim.x * blockIdx.x + threadIdx.x;</div>
<div class="line"><a id="l01201" name="l01201"></a><span class="lineno"> 1201</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> warpIdx = tid / WARP_SIZE;</div>
<div class="line"><a id="l01202" name="l01202"></a><span class="lineno"> 1202</span>        <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> laneIdx = tid % WARP_SIZE;</div>
<div class="line"><a id="l01203" name="l01203"></a><span class="lineno"> 1203</span>        <span class="keywordtype">float</span> localSum = 0.0f;</div>
<div class="line"><a id="l01204" name="l01204"></a><span class="lineno"> 1204</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l01205" name="l01205"></a><span class="lineno"> 1205</span>            localSum = in[idx + offset];</div>
<div class="line"><a id="l01206" name="l01206"></a><span class="lineno"> 1206</span>        }</div>
<div class="line"><a id="l01207" name="l01207"></a><span class="lineno"> 1207</span>        __syncthreads();</div>
<div class="line"><a id="l01208" name="l01208"></a><span class="lineno"> 1208</span>        localSum = warpReduce(localSum);</div>
<div class="line"><a id="l01209" name="l01209"></a><span class="lineno"> 1209</span>        __syncthreads();</div>
<div class="line"><a id="l01210" name="l01210"></a><span class="lineno"> 1210</span>        <span class="keywordflow">if</span> (laneIdx == 0) {</div>
<div class="line"><a id="l01211" name="l01211"></a><span class="lineno"> 1211</span>            sdata[warpIdx] = localSum;</div>
<div class="line"><a id="l01212" name="l01212"></a><span class="lineno"> 1212</span>        }</div>
<div class="line"><a id="l01213" name="l01213"></a><span class="lineno"> 1213</span>        __syncthreads();</div>
<div class="line"><a id="l01214" name="l01214"></a><span class="lineno"> 1214</span>        localSum = (tid &lt; blockDim.x / WARP_SIZE) ? sdata[laneIdx] : 0.0f;</div>
<div class="line"><a id="l01215" name="l01215"></a><span class="lineno"> 1215</span>        __syncthreads();</div>
<div class="line"><a id="l01216" name="l01216"></a><span class="lineno"> 1216</span>        <span class="keywordflow">if</span> (warpIdx == 0) {</div>
<div class="line"><a id="l01217" name="l01217"></a><span class="lineno"> 1217</span>            localSum = warpReduce(localSum);</div>
<div class="line"><a id="l01218" name="l01218"></a><span class="lineno"> 1218</span>        }</div>
<div class="line"><a id="l01219" name="l01219"></a><span class="lineno"> 1219</span>        __syncthreads();</div>
<div class="line"><a id="l01220" name="l01220"></a><span class="lineno"> 1220</span>        <span class="keywordflow">if</span> (tid == 0) {</div>
<div class="line"><a id="l01221" name="l01221"></a><span class="lineno"> 1221</span>            out[blockIdx.x] = localSum;</div>
<div class="line"><a id="l01222" name="l01222"></a><span class="lineno"> 1222</span>        }</div>
<div class="line"><a id="l01223" name="l01223"></a><span class="lineno"> 1223</span>    }</div>
<div class="line"><a id="l01224" name="l01224"></a><span class="lineno"> 1224</span> </div>
<div class="foldopen" id="foldopen01225" data-start="{" data-end="}">
<div class="line"><a id="l01225" name="l01225"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a1ae846a65c2f5b83cd1b9fc61b877854"> 1225</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a1ae846a65c2f5b83cd1b9fc61b877854">Summation</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> sharedMemSize, <span class="keywordtype">float</span>* out,</div>
<div class="line"><a id="l01226" name="l01226"></a><span class="lineno"> 1226</span>                   <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">unsigned</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset) {</div>
<div class="line"><a id="l01227" name="l01227"></a><span class="lineno"> 1227</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(SummationKernel, gridDim, blockDim, sharedMemSize, out, in, n, offset);</div>
<div class="line"><a id="l01228" name="l01228"></a><span class="lineno"> 1228</span>    }</div>
</div>
<div class="line"><a id="l01229" name="l01229"></a><span class="lineno"> 1229</span> </div>
<div class="line"><a id="l01230" name="l01230"></a><span class="lineno"> 1230</span>    __global__ <span class="keywordtype">void</span> gradCopyKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_o,</div>
<div class="line"><a id="l01231" name="l01231"></a><span class="lineno"> 1231</span>                                   <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_i) {</div>
<div class="line"><a id="l01232" name="l01232"></a><span class="lineno"> 1232</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01233" name="l01233"></a><span class="lineno"> 1233</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l01234" name="l01234"></a><span class="lineno"> 1234</span>            out[idx + offset_o] += in[idx + offset_i];</div>
<div class="line"><a id="l01235" name="l01235"></a><span class="lineno"> 1235</span>        }</div>
<div class="line"><a id="l01236" name="l01236"></a><span class="lineno"> 1236</span>    }</div>
<div class="line"><a id="l01237" name="l01237"></a><span class="lineno"> 1237</span> </div>
<div class="foldopen" id="foldopen01238" data-start="{" data-end="}">
<div class="line"><a id="l01238" name="l01238"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a0ed44a68bfb86a9fd3d6c3b25614713f"> 1238</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a0ed44a68bfb86a9fd3d6c3b25614713f">gradCopy</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> n,</div>
<div class="line"><a id="l01239" name="l01239"></a><span class="lineno"> 1239</span>                  <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_o, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_i) {</div>
<div class="line"><a id="l01240" name="l01240"></a><span class="lineno"> 1240</span>        <span class="keywordflow">if</span> (offset_o.size() != offset_i.size()) {</div>
<div class="line"><a id="l01241" name="l01241"></a><span class="lineno"> 1241</span>            <span class="keywordflow">throw</span> std::invalid_argument(<span class="stringliteral">&quot;offset size do not match&quot;</span>);</div>
<div class="line"><a id="l01242" name="l01242"></a><span class="lineno"> 1242</span>        }</div>
<div class="line"><a id="l01243" name="l01243"></a><span class="lineno"> 1243</span>        std::vector&lt;cudaStream_t&gt; streams;</div>
<div class="line"><a id="l01244" name="l01244"></a><span class="lineno"> 1244</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset_i.size(); i++) {</div>
<div class="line"><a id="l01245" name="l01245"></a><span class="lineno"> 1245</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l01246" name="l01246"></a><span class="lineno"> 1246</span>            streams.push_back(stream);</div>
<div class="line"><a id="l01247" name="l01247"></a><span class="lineno"> 1247</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(out, stream);</div>
<div class="line"><a id="l01248" name="l01248"></a><span class="lineno"> 1248</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(in, stream);</div>
<div class="line"><a id="l01249" name="l01249"></a><span class="lineno"> 1249</span>            gradCopyKernel&lt;&lt;&lt;gridDim, blockDim, 0, stream&gt;&gt;&gt;(out, in, n, offset_o[i], offset_i[i]);</div>
<div class="line"><a id="l01250" name="l01250"></a><span class="lineno"> 1250</span>        }</div>
<div class="line"><a id="l01251" name="l01251"></a><span class="lineno"> 1251</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l01252" name="l01252"></a><span class="lineno"> 1252</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(out, stream);</div>
<div class="line"><a id="l01253" name="l01253"></a><span class="lineno"> 1253</span>        }</div>
<div class="line"><a id="l01254" name="l01254"></a><span class="lineno"> 1254</span>    }</div>
</div>
<div class="line"><a id="l01255" name="l01255"></a><span class="lineno"> 1255</span> </div>
<div class="line"><a id="l01256" name="l01256"></a><span class="lineno"> 1256</span>    __global__ <span class="keywordtype">void</span> NgradCopyKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_o,</div>
<div class="line"><a id="l01257" name="l01257"></a><span class="lineno"> 1257</span>                                    <span class="keyword">const</span> <span class="keywordtype">size_t</span> offset_i) {</div>
<div class="line"><a id="l01258" name="l01258"></a><span class="lineno"> 1258</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01259" name="l01259"></a><span class="lineno"> 1259</span>        <span class="keywordflow">if</span> (idx &lt; n) {</div>
<div class="line"><a id="l01260" name="l01260"></a><span class="lineno"> 1260</span>            out[idx + offset_o] -= in[idx + offset_i];</div>
<div class="line"><a id="l01261" name="l01261"></a><span class="lineno"> 1261</span>        }</div>
<div class="line"><a id="l01262" name="l01262"></a><span class="lineno"> 1262</span>    }</div>
<div class="line"><a id="l01263" name="l01263"></a><span class="lineno"> 1263</span> </div>
<div class="foldopen" id="foldopen01264" data-start="{" data-end="}">
<div class="line"><a id="l01264" name="l01264"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a9ac0590fbb5eb7f51b05da574e9845a8"> 1264</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a9ac0590fbb5eb7f51b05da574e9845a8">NgradCopy</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> n,</div>
<div class="line"><a id="l01265" name="l01265"></a><span class="lineno"> 1265</span>                   <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_o, <span class="keyword">const</span> std::vector&lt;size_t&gt;&amp; offset_i) {</div>
<div class="line"><a id="l01266" name="l01266"></a><span class="lineno"> 1266</span>        <span class="keywordflow">if</span> (offset_o.size() != offset_i.size()) {</div>
<div class="line"><a id="l01267" name="l01267"></a><span class="lineno"> 1267</span>            <span class="keywordflow">throw</span> std::invalid_argument(<span class="stringliteral">&quot;offset size do not match&quot;</span>);</div>
<div class="line"><a id="l01268" name="l01268"></a><span class="lineno"> 1268</span>        }</div>
<div class="line"><a id="l01269" name="l01269"></a><span class="lineno"> 1269</span>        std::vector&lt;cudaStream_t&gt; streams;</div>
<div class="line"><a id="l01270" name="l01270"></a><span class="lineno"> 1270</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> i = 0; i &lt; offset_i.size(); i++) {</div>
<div class="line"><a id="l01271" name="l01271"></a><span class="lineno"> 1271</span>            cudaStream_t stream = <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">getStream</a>();</div>
<div class="line"><a id="l01272" name="l01272"></a><span class="lineno"> 1272</span>            streams.push_back(stream);</div>
<div class="line"><a id="l01273" name="l01273"></a><span class="lineno"> 1273</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(out, stream);</div>
<div class="line"><a id="l01274" name="l01274"></a><span class="lineno"> 1274</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">streamWait</a>(in, stream);</div>
<div class="line"><a id="l01275" name="l01275"></a><span class="lineno"> 1275</span>            NgradCopyKernel&lt;&lt;&lt;gridDim, blockDim, 0, stream&gt;&gt;&gt;(out, in, n, offset_o[i], offset_i[i]);</div>
<div class="line"><a id="l01276" name="l01276"></a><span class="lineno"> 1276</span>        }</div>
<div class="line"><a id="l01277" name="l01277"></a><span class="lineno"> 1277</span>        <span class="keywordflow">for</span> (<span class="keyword">auto</span> stream : streams) {</div>
<div class="line"><a id="l01278" name="l01278"></a><span class="lineno"> 1278</span>            <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">recordData</a>(out, stream);</div>
<div class="line"><a id="l01279" name="l01279"></a><span class="lineno"> 1279</span>        }</div>
<div class="line"><a id="l01280" name="l01280"></a><span class="lineno"> 1280</span>    }</div>
</div>
<div class="line"><a id="l01281" name="l01281"></a><span class="lineno"> 1281</span> </div>
<div class="line"><a id="l01282" name="l01282"></a><span class="lineno"> 1282</span>    __global__ <span class="keywordtype">void</span> ExpandKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> total) {</div>
<div class="line"><a id="l01283" name="l01283"></a><span class="lineno"> 1283</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01284" name="l01284"></a><span class="lineno"> 1284</span>        <span class="keywordflow">if</span> (idx &gt;= total) {</div>
<div class="line"><a id="l01285" name="l01285"></a><span class="lineno"> 1285</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01286" name="l01286"></a><span class="lineno"> 1286</span>        }</div>
<div class="line"><a id="l01287" name="l01287"></a><span class="lineno"> 1287</span>        out[idx] = in[idx % n];</div>
<div class="line"><a id="l01288" name="l01288"></a><span class="lineno"> 1288</span>    }</div>
<div class="line"><a id="l01289" name="l01289"></a><span class="lineno"> 1289</span> </div>
<div class="foldopen" id="foldopen01290" data-start="{" data-end="}">
<div class="line"><a id="l01290" name="l01290"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#ae45dbebceb76ddf82fa5e6b9df882e62"> 1290</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#ae45dbebceb76ddf82fa5e6b9df882e62">Expand</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> n,</div>
<div class="line"><a id="l01291" name="l01291"></a><span class="lineno"> 1291</span>                <span class="keyword">const</span> <span class="keywordtype">size_t</span> total) {</div>
<div class="line"><a id="l01292" name="l01292"></a><span class="lineno"> 1292</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(ExpandKernel, gridDim, blockDim, 0, out, in, n, total);</div>
<div class="line"><a id="l01293" name="l01293"></a><span class="lineno"> 1293</span>    }</div>
</div>
<div class="line"><a id="l01294" name="l01294"></a><span class="lineno"> 1294</span> </div>
<div class="line"><a id="l01295" name="l01295"></a><span class="lineno"> 1295</span>    __global__ <span class="keywordtype">void</span> CompressKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> n, <span class="keyword">const</span> <span class="keywordtype">size_t</span> total) {</div>
<div class="line"><a id="l01296" name="l01296"></a><span class="lineno"> 1296</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01297" name="l01297"></a><span class="lineno"> 1297</span>        <span class="keywordflow">if</span> (idx &gt;= total) {</div>
<div class="line"><a id="l01298" name="l01298"></a><span class="lineno"> 1298</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01299" name="l01299"></a><span class="lineno"> 1299</span>        }</div>
<div class="line"><a id="l01300" name="l01300"></a><span class="lineno"> 1300</span>        atomicAdd(out + idx % n, in[idx]);</div>
<div class="line"><a id="l01301" name="l01301"></a><span class="lineno"> 1301</span>    }</div>
<div class="line"><a id="l01302" name="l01302"></a><span class="lineno"> 1302</span> </div>
<div class="foldopen" id="foldopen01303" data-start="{" data-end="}">
<div class="line"><a id="l01303" name="l01303"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a454a28ef0e22014efca1ede4e954db65"> 1303</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a454a28ef0e22014efca1ede4e954db65">Compress</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> n,</div>
<div class="line"><a id="l01304" name="l01304"></a><span class="lineno"> 1304</span>                  <span class="keyword">const</span> <span class="keywordtype">size_t</span> total) {</div>
<div class="line"><a id="l01305" name="l01305"></a><span class="lineno"> 1305</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(CompressKernel, gridDim, blockDim, 0, out, in, n, total);</div>
<div class="line"><a id="l01306" name="l01306"></a><span class="lineno"> 1306</span>    }</div>
</div>
<div class="line"><a id="l01307" name="l01307"></a><span class="lineno"> 1307</span> </div>
<div class="line"><a id="l01308" name="l01308"></a><span class="lineno"> 1308</span>    __global__ <span class="keywordtype">void</span> img2colKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> C,</div>
<div class="line"><a id="l01309" name="l01309"></a><span class="lineno"> 1309</span>                                  <span class="keyword">const</span> <span class="keywordtype">size_t</span> K_h, <span class="keyword">const</span> <span class="keywordtype">size_t</span> K_w, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> pad,</div>
<div class="line"><a id="l01310" name="l01310"></a><span class="lineno"> 1310</span>                                  <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batch) {</div>
<div class="line"><a id="l01311" name="l01311"></a><span class="lineno"> 1311</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01312" name="l01312"></a><span class="lineno"> 1312</span>        <span class="keywordflow">if</span> (idx &gt;= H_out * W_out * C * K_h * K_w * batch) {</div>
<div class="line"><a id="l01313" name="l01313"></a><span class="lineno"> 1313</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01314" name="l01314"></a><span class="lineno"> 1314</span>        }</div>
<div class="line"><a id="l01315" name="l01315"></a><span class="lineno"> 1315</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> fixedIdx = idx % (H_out * W_out * C * K_h * K_w);</div>
<div class="line"><a id="l01316" name="l01316"></a><span class="lineno"> 1316</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentBatch = idx / (H_out * W_out * C * K_h * K_w);</div>
<div class="line"><a id="l01317" name="l01317"></a><span class="lineno"> 1317</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> k = fixedIdx / (C * K_h * K_w);</div>
<div class="line"><a id="l01318" name="l01318"></a><span class="lineno"> 1318</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> m = fixedIdx % (C * K_h * K_w);</div>
<div class="line"><a id="l01319" name="l01319"></a><span class="lineno"> 1319</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> c = m / (K_h * K_w);</div>
<div class="line"><a id="l01320" name="l01320"></a><span class="lineno"> 1320</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h = (k / W_out) * stride - pad + (m % (K_h * K_w)) / K_w;</div>
<div class="line"><a id="l01321" name="l01321"></a><span class="lineno"> 1321</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w = (k % W_out) * stride - pad + m % K_w;</div>
<div class="line"><a id="l01322" name="l01322"></a><span class="lineno"> 1322</span>        <span class="keywordflow">if</span> (h &gt;= 0 &amp;&amp; h &lt; H_in &amp;&amp; w &gt;= 0 &amp;&amp; w &lt; W_in) {</div>
<div class="line"><a id="l01323" name="l01323"></a><span class="lineno"> 1323</span>            out[idx] = in[currentBatch * (C * H_in * W_in) + c * (H_in * W_in) + h * W_in + w];</div>
<div class="line"><a id="l01324" name="l01324"></a><span class="lineno"> 1324</span>        }</div>
<div class="line"><a id="l01325" name="l01325"></a><span class="lineno"> 1325</span>        <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l01326" name="l01326"></a><span class="lineno"> 1326</span>            out[idx] = 0;</div>
<div class="line"><a id="l01327" name="l01327"></a><span class="lineno"> 1327</span>        }</div>
<div class="line"><a id="l01328" name="l01328"></a><span class="lineno"> 1328</span>    }</div>
<div class="line"><a id="l01329" name="l01329"></a><span class="lineno"> 1329</span> </div>
<div class="foldopen" id="foldopen01330" data-start="{" data-end="}">
<div class="line"><a id="l01330" name="l01330"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a3a781324400c54c35dd564f3599dca8e"> 1330</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a3a781324400c54c35dd564f3599dca8e">img2col</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out,</div>
<div class="line"><a id="l01331" name="l01331"></a><span class="lineno"> 1331</span>                 <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> C, <span class="keyword">const</span> <span class="keywordtype">size_t</span> K_h, <span class="keyword">const</span> <span class="keywordtype">size_t</span> K_w, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride,</div>
<div class="line"><a id="l01332" name="l01332"></a><span class="lineno"> 1332</span>                 <span class="keyword">const</span> <span class="keywordtype">size_t</span> pad, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batch) {</div>
<div class="line"><a id="l01333" name="l01333"></a><span class="lineno"> 1333</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(img2colKernel, gridDim, blockDim, 0, out, in, H_out, W_out, C,</div>
<div class="line"><a id="l01334" name="l01334"></a><span class="lineno"> 1334</span>                                                K_h, K_w, stride, pad, H_in, W_in, batch);</div>
<div class="line"><a id="l01335" name="l01335"></a><span class="lineno"> 1335</span>    }</div>
</div>
<div class="line"><a id="l01336" name="l01336"></a><span class="lineno"> 1336</span> </div>
<div class="line"><a id="l01337" name="l01337"></a><span class="lineno"> 1337</span>    __global__ <span class="keywordtype">void</span> img2colBackwardKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out,</div>
<div class="line"><a id="l01338" name="l01338"></a><span class="lineno"> 1338</span>                                          <span class="keyword">const</span> <span class="keywordtype">size_t</span> C,</div>
<div class="line"><a id="l01339" name="l01339"></a><span class="lineno"> 1339</span>                                          <span class="keyword">const</span> <span class="keywordtype">size_t</span> K_h, <span class="keyword">const</span> <span class="keywordtype">size_t</span> K_w, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> pad,</div>
<div class="line"><a id="l01340" name="l01340"></a><span class="lineno"> 1340</span>                                          <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batch) {</div>
<div class="line"><a id="l01341" name="l01341"></a><span class="lineno"> 1341</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01342" name="l01342"></a><span class="lineno"> 1342</span>        <span class="keywordflow">if</span> (idx &gt;= H_out * W_out * C * K_h * K_w * batch) {</div>
<div class="line"><a id="l01343" name="l01343"></a><span class="lineno"> 1343</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01344" name="l01344"></a><span class="lineno"> 1344</span>        }</div>
<div class="line"><a id="l01345" name="l01345"></a><span class="lineno"> 1345</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> fixedIdx = idx % (H_out * W_out * C * K_h * K_w);</div>
<div class="line"><a id="l01346" name="l01346"></a><span class="lineno"> 1346</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentBatch = idx / (H_out * W_out * C * K_h * K_w);</div>
<div class="line"><a id="l01347" name="l01347"></a><span class="lineno"> 1347</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> k = fixedIdx / (C * K_h * K_w);</div>
<div class="line"><a id="l01348" name="l01348"></a><span class="lineno"> 1348</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> m = fixedIdx % (C * K_h * K_w);</div>
<div class="line"><a id="l01349" name="l01349"></a><span class="lineno"> 1349</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> c = m / (K_h * K_w);</div>
<div class="line"><a id="l01350" name="l01350"></a><span class="lineno"> 1350</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h = (k / W_out) * stride - pad + (m % (K_h * K_w)) / K_w;</div>
<div class="line"><a id="l01351" name="l01351"></a><span class="lineno"> 1351</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w = (k % W_out) * stride - pad + m % K_w;</div>
<div class="line"><a id="l01352" name="l01352"></a><span class="lineno"> 1352</span>        <span class="keywordflow">if</span> (h &gt;= 0 &amp;&amp; h &lt; H_in &amp;&amp; w &gt;= 0 &amp;&amp; w &lt; W_in) {</div>
<div class="line"><a id="l01353" name="l01353"></a><span class="lineno"> 1353</span>            atomicAdd(out + currentBatch * (C * H_in * W_in) + c * (H_in * W_in) + h * W_in + w, in[idx]);</div>
<div class="line"><a id="l01354" name="l01354"></a><span class="lineno"> 1354</span>        }</div>
<div class="line"><a id="l01355" name="l01355"></a><span class="lineno"> 1355</span>    }</div>
<div class="line"><a id="l01356" name="l01356"></a><span class="lineno"> 1356</span> </div>
<div class="foldopen" id="foldopen01357" data-start="{" data-end="}">
<div class="line"><a id="l01357" name="l01357"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a1c2b7a6f28d2af22f9a2623c5ae62bff"> 1357</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a1c2b7a6f28d2af22f9a2623c5ae62bff">img2colBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out,</div>
<div class="line"><a id="l01358" name="l01358"></a><span class="lineno"> 1358</span>                         <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> C, <span class="keyword">const</span> <span class="keywordtype">size_t</span> K_h, <span class="keyword">const</span> <span class="keywordtype">size_t</span> K_w, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride,</div>
<div class="line"><a id="l01359" name="l01359"></a><span class="lineno"> 1359</span>                         <span class="keyword">const</span> <span class="keywordtype">size_t</span> pad, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batch) {</div>
<div class="line"><a id="l01360" name="l01360"></a><span class="lineno"> 1360</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(img2colBackwardKernel, gridDim, blockDim, 0, out, in, H_out,</div>
<div class="line"><a id="l01361" name="l01361"></a><span class="lineno"> 1361</span>                                                W_out, C, K_h, K_w, stride, pad, H_in, W_in, batch);</div>
<div class="line"><a id="l01362" name="l01362"></a><span class="lineno"> 1362</span>    }</div>
</div>
<div class="line"><a id="l01363" name="l01363"></a><span class="lineno"> 1363</span> </div>
<div class="line"><a id="l01364" name="l01364"></a><span class="lineno"> 1364</span>    __global__ <span class="keywordtype">void</span> col2imgKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out,</div>
<div class="line"><a id="l01365" name="l01365"></a><span class="lineno"> 1365</span>                                  <span class="keyword">const</span> <span class="keywordtype">size_t</span> C_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches) {</div>
<div class="line"><a id="l01366" name="l01366"></a><span class="lineno"> 1366</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockDim.x * blockIdx.x + threadIdx.x;</div>
<div class="line"><a id="l01367" name="l01367"></a><span class="lineno"> 1367</span>        <span class="keywordflow">if</span> (idx &gt;= H_out * W_out * C_out * batches) {</div>
<div class="line"><a id="l01368" name="l01368"></a><span class="lineno"> 1368</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01369" name="l01369"></a><span class="lineno"> 1369</span>        }</div>
<div class="line"><a id="l01370" name="l01370"></a><span class="lineno"> 1370</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> batch = idx / (C_out * H_out * W_out);</div>
<div class="line"><a id="l01371" name="l01371"></a><span class="lineno"> 1371</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> fixedIdx = idx % (C_out * H_out * W_out);</div>
<div class="line"><a id="l01372" name="l01372"></a><span class="lineno"> 1372</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> c = fixedIdx / (H_out * W_out);</div>
<div class="line"><a id="l01373" name="l01373"></a><span class="lineno"> 1373</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> h = (fixedIdx % (H_out * W_out)) / W_out;</div>
<div class="line"><a id="l01374" name="l01374"></a><span class="lineno"> 1374</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> w = (fixedIdx % (H_out * W_out)) % W_out;</div>
<div class="line"><a id="l01375" name="l01375"></a><span class="lineno"> 1375</span>        out[idx] = in[batch * (C_out * H_out * W_out) + (h * W_out + w) * C_out + c];</div>
<div class="line"><a id="l01376" name="l01376"></a><span class="lineno"> 1376</span>    }</div>
<div class="line"><a id="l01377" name="l01377"></a><span class="lineno"> 1377</span> </div>
<div class="foldopen" id="foldopen01378" data-start="{" data-end="}">
<div class="line"><a id="l01378" name="l01378"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a7c061f5511c3ab9d36563757bd969ff7"> 1378</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a7c061f5511c3ab9d36563757bd969ff7">col2img</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out,</div>
<div class="line"><a id="l01379" name="l01379"></a><span class="lineno"> 1379</span>                 <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> C_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches) {</div>
<div class="line"><a id="l01380" name="l01380"></a><span class="lineno"> 1380</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(col2imgKernel, gridDim, blockDim, 0, out, in, H_out, W_out, C_out,</div>
<div class="line"><a id="l01381" name="l01381"></a><span class="lineno"> 1381</span>                                                batches);</div>
<div class="line"><a id="l01382" name="l01382"></a><span class="lineno"> 1382</span>    }</div>
</div>
<div class="line"><a id="l01383" name="l01383"></a><span class="lineno"> 1383</span> </div>
<div class="line"><a id="l01384" name="l01384"></a><span class="lineno"> 1384</span>    __global__ <span class="keywordtype">void</span> col2imgBackwardKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out,</div>
<div class="line"><a id="l01385" name="l01385"></a><span class="lineno"> 1385</span>                                          <span class="keyword">const</span> <span class="keywordtype">size_t</span> C_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches) {</div>
<div class="line"><a id="l01386" name="l01386"></a><span class="lineno"> 1386</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockDim.x * blockIdx.x + threadIdx.x;</div>
<div class="line"><a id="l01387" name="l01387"></a><span class="lineno"> 1387</span>        <span class="keywordflow">if</span> (idx &gt;= H_out * W_out * C_out * batches) {</div>
<div class="line"><a id="l01388" name="l01388"></a><span class="lineno"> 1388</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01389" name="l01389"></a><span class="lineno"> 1389</span>        }</div>
<div class="line"><a id="l01390" name="l01390"></a><span class="lineno"> 1390</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> batch = idx / (C_out * H_out * W_out);</div>
<div class="line"><a id="l01391" name="l01391"></a><span class="lineno"> 1391</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> fixedIdx = idx % (C_out * H_out * W_out);</div>
<div class="line"><a id="l01392" name="l01392"></a><span class="lineno"> 1392</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> c = fixedIdx / (H_out * W_out);</div>
<div class="line"><a id="l01393" name="l01393"></a><span class="lineno"> 1393</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> h = (fixedIdx % (H_out * W_out)) / W_out;</div>
<div class="line"><a id="l01394" name="l01394"></a><span class="lineno"> 1394</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> w = (fixedIdx % (H_out * W_out)) % W_out;</div>
<div class="line"><a id="l01395" name="l01395"></a><span class="lineno"> 1395</span>        out[batch * (C_out * H_out * W_out) + (h * W_out + w) * C_out + c] = in[idx];</div>
<div class="line"><a id="l01396" name="l01396"></a><span class="lineno"> 1396</span>    }</div>
<div class="line"><a id="l01397" name="l01397"></a><span class="lineno"> 1397</span> </div>
<div class="foldopen" id="foldopen01398" data-start="{" data-end="}">
<div class="line"><a id="l01398" name="l01398"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a028970809074d79f28ff94f62b3edaa4"> 1398</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a028970809074d79f28ff94f62b3edaa4">col2imgBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out,</div>
<div class="line"><a id="l01399" name="l01399"></a><span class="lineno"> 1399</span>                         <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> C_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches) {</div>
<div class="line"><a id="l01400" name="l01400"></a><span class="lineno"> 1400</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(col2imgBackwardKernel, gridDim, blockDim, 0, out, in, H_out, W_out,</div>
<div class="line"><a id="l01401" name="l01401"></a><span class="lineno"> 1401</span>                                                C_out, batches);</div>
<div class="line"><a id="l01402" name="l01402"></a><span class="lineno"> 1402</span>    }</div>
</div>
<div class="line"><a id="l01403" name="l01403"></a><span class="lineno"> 1403</span> </div>
<div class="line"><a id="l01404" name="l01404"></a><span class="lineno"> 1404</span>    __global__ <span class="keywordtype">void</span> AveragePoolingKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> pool_size, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> padding,</div>
<div class="line"><a id="l01405" name="l01405"></a><span class="lineno"> 1405</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out) {</div>
<div class="line"><a id="l01406" name="l01406"></a><span class="lineno"> 1406</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01407" name="l01407"></a><span class="lineno"> 1407</span>        <span class="keywordflow">if</span> (idx &gt;= batches * channels * H_out * W_out) {</div>
<div class="line"><a id="l01408" name="l01408"></a><span class="lineno"> 1408</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01409" name="l01409"></a><span class="lineno"> 1409</span>        }</div>
<div class="line"><a id="l01410" name="l01410"></a><span class="lineno"> 1410</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentBatch = idx / (channels * H_out * W_out);</div>
<div class="line"><a id="l01411" name="l01411"></a><span class="lineno"> 1411</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentChannel = (idx % (channels * H_out * W_out)) / (H_out * W_out);</div>
<div class="line"><a id="l01412" name="l01412"></a><span class="lineno"> 1412</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> h = (idx % (H_out * W_out)) / W_out;</div>
<div class="line"><a id="l01413" name="l01413"></a><span class="lineno"> 1413</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> w = (idx % (H_out * W_out)) % W_out;</div>
<div class="line"><a id="l01414" name="l01414"></a><span class="lineno"> 1414</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_start = h * stride - padding;</div>
<div class="line"><a id="l01415" name="l01415"></a><span class="lineno"> 1415</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_start = w * stride - padding;</div>
<div class="line"><a id="l01416" name="l01416"></a><span class="lineno"> 1416</span>        out[idx] = 0.0f;</div>
<div class="line"><a id="l01417" name="l01417"></a><span class="lineno"> 1417</span>        <span class="keywordtype">size_t</span> count = 0;</div>
<div class="line"><a id="l01418" name="l01418"></a><span class="lineno"> 1418</span>        <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> i = 0; i &lt; pool_size; i++) {</div>
<div class="line"><a id="l01419" name="l01419"></a><span class="lineno"> 1419</span>            <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> j = 0; j &lt; pool_size; j++) {</div>
<div class="line"><a id="l01420" name="l01420"></a><span class="lineno"> 1420</span>                <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_in = h_start + i;</div>
<div class="line"><a id="l01421" name="l01421"></a><span class="lineno"> 1421</span>                <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_in = w_start + j;</div>
<div class="line"><a id="l01422" name="l01422"></a><span class="lineno"> 1422</span>                <span class="keywordflow">if</span> (h_in &gt;= 0 &amp;&amp; h_in &lt; H_in &amp;&amp; w_in &gt;= 0 &amp;&amp; w_in &lt; W_in) {</div>
<div class="line"><a id="l01423" name="l01423"></a><span class="lineno"> 1423</span>                    out[idx] += in[currentBatch * (channels * H_in * W_in) + currentChannel * (H_in * W_in) + h_in * W_in + w_in];</div>
<div class="line"><a id="l01424" name="l01424"></a><span class="lineno"> 1424</span>                    count++;</div>
<div class="line"><a id="l01425" name="l01425"></a><span class="lineno"> 1425</span>                }</div>
<div class="line"><a id="l01426" name="l01426"></a><span class="lineno"> 1426</span>            }</div>
<div class="line"><a id="l01427" name="l01427"></a><span class="lineno"> 1427</span>        }</div>
<div class="line"><a id="l01428" name="l01428"></a><span class="lineno"> 1428</span>        out[idx] = count &gt; 0 ? out[idx] / (float)count : 0.0f;</div>
<div class="line"><a id="l01429" name="l01429"></a><span class="lineno"> 1429</span>    }</div>
<div class="line"><a id="l01430" name="l01430"></a><span class="lineno"> 1430</span> </div>
<div class="foldopen" id="foldopen01431" data-start="{" data-end="}">
<div class="line"><a id="l01431" name="l01431"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#addaa377a94d007df2690043b08904e28"> 1431</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#addaa377a94d007df2690043b08904e28">AveragePooling</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l01432" name="l01432"></a><span class="lineno"> 1432</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> pool_size, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> padding,</div>
<div class="line"><a id="l01433" name="l01433"></a><span class="lineno"> 1433</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in,</div>
<div class="line"><a id="l01434" name="l01434"></a><span class="lineno"> 1434</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out) {</div>
<div class="line"><a id="l01435" name="l01435"></a><span class="lineno"> 1435</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(AveragePoolingKernel, gridDim, blockDim, 0, out, in,</div>
<div class="line"><a id="l01436" name="l01436"></a><span class="lineno"> 1436</span>            pool_size, stride, padding, batches, channels, H_in, W_in, H_out, W_out);</div>
<div class="line"><a id="l01437" name="l01437"></a><span class="lineno"> 1437</span>    }</div>
</div>
<div class="line"><a id="l01438" name="l01438"></a><span class="lineno"> 1438</span> </div>
<div class="line"><a id="l01439" name="l01439"></a><span class="lineno"> 1439</span>    __global__ <span class="keywordtype">void</span> AveragePoolingBackwardKernel(<span class="keywordtype">float</span>* out, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> pool_size, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> padding,</div>
<div class="line"><a id="l01440" name="l01440"></a><span class="lineno"> 1440</span>    <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out) {</div>
<div class="line"><a id="l01441" name="l01441"></a><span class="lineno"> 1441</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01442" name="l01442"></a><span class="lineno"> 1442</span>        <span class="keywordflow">if</span> (idx &gt;= batches * channels * H_out * W_out) {</div>
<div class="line"><a id="l01443" name="l01443"></a><span class="lineno"> 1443</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01444" name="l01444"></a><span class="lineno"> 1444</span>        }</div>
<div class="line"><a id="l01445" name="l01445"></a><span class="lineno"> 1445</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentBatch = idx / (channels * H_out * W_out);</div>
<div class="line"><a id="l01446" name="l01446"></a><span class="lineno"> 1446</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentChannel = (idx % (channels * H_out * W_out)) / (H_out * W_out);</div>
<div class="line"><a id="l01447" name="l01447"></a><span class="lineno"> 1447</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> h = (idx % (H_out * W_out)) / W_out;</div>
<div class="line"><a id="l01448" name="l01448"></a><span class="lineno"> 1448</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> w = (idx % (H_out * W_out)) % W_out;</div>
<div class="line"><a id="l01449" name="l01449"></a><span class="lineno"> 1449</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_start = h * stride - padding;</div>
<div class="line"><a id="l01450" name="l01450"></a><span class="lineno"> 1450</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_start = w * stride - padding;</div>
<div class="line"><a id="l01451" name="l01451"></a><span class="lineno"> 1451</span>        <span class="keywordflow">if</span> (!padding) {</div>
<div class="line"><a id="l01452" name="l01452"></a><span class="lineno"> 1452</span>            <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> i = 0; i &lt; pool_size; i++) {</div>
<div class="line"><a id="l01453" name="l01453"></a><span class="lineno"> 1453</span>                <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> j = 0; j &lt; pool_size; j++) {</div>
<div class="line"><a id="l01454" name="l01454"></a><span class="lineno"> 1454</span>                    <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_in = h_start + i;</div>
<div class="line"><a id="l01455" name="l01455"></a><span class="lineno"> 1455</span>                    <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_in = w_start + j;</div>
<div class="line"><a id="l01456" name="l01456"></a><span class="lineno"> 1456</span>                    <span class="keywordflow">if</span> (h_in &gt;= 0 &amp;&amp; h_in &lt; H_in &amp;&amp; w_in &gt;= 0 &amp;&amp; w_in &lt; W_in) {</div>
<div class="line"><a id="l01457" name="l01457"></a><span class="lineno"> 1457</span>                        atomicAdd(out + currentBatch * (channels * H_in * W_in) + currentChannel * (H_in * W_in) + h_in * W_in + w_in, in[idx] / (<span class="keywordtype">float</span>)(pool_size*pool_size));</div>
<div class="line"><a id="l01458" name="l01458"></a><span class="lineno"> 1458</span>                    }</div>
<div class="line"><a id="l01459" name="l01459"></a><span class="lineno"> 1459</span>                }</div>
<div class="line"><a id="l01460" name="l01460"></a><span class="lineno"> 1460</span>            }</div>
<div class="line"><a id="l01461" name="l01461"></a><span class="lineno"> 1461</span>        } <span class="keywordflow">else</span> {</div>
<div class="line"><a id="l01462" name="l01462"></a><span class="lineno"> 1462</span>            <span class="keywordtype">size_t</span> count = 0;</div>
<div class="line"><a id="l01463" name="l01463"></a><span class="lineno"> 1463</span>            <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> i = 0; i &lt; pool_size; i++) {</div>
<div class="line"><a id="l01464" name="l01464"></a><span class="lineno"> 1464</span>                <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> j = 0; j &lt; pool_size; j++) {</div>
<div class="line"><a id="l01465" name="l01465"></a><span class="lineno"> 1465</span>                    <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_in = h_start + i;</div>
<div class="line"><a id="l01466" name="l01466"></a><span class="lineno"> 1466</span>                    <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_in = w_start + j;</div>
<div class="line"><a id="l01467" name="l01467"></a><span class="lineno"> 1467</span>                    <span class="keywordflow">if</span> (h_in &gt;= 0 &amp;&amp; h_in &lt; H_in &amp;&amp; w_in &gt;= 0 &amp;&amp; w_in &lt; W_in) {</div>
<div class="line"><a id="l01468" name="l01468"></a><span class="lineno"> 1468</span>                        count++;</div>
<div class="line"><a id="l01469" name="l01469"></a><span class="lineno"> 1469</span>                    }</div>
<div class="line"><a id="l01470" name="l01470"></a><span class="lineno"> 1470</span>                }</div>
<div class="line"><a id="l01471" name="l01471"></a><span class="lineno"> 1471</span>            }</div>
<div class="line"><a id="l01472" name="l01472"></a><span class="lineno"> 1472</span>            <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> i = 0; i &lt; pool_size; i++) {</div>
<div class="line"><a id="l01473" name="l01473"></a><span class="lineno"> 1473</span>                <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> j = 0; j &lt; pool_size; j++) {</div>
<div class="line"><a id="l01474" name="l01474"></a><span class="lineno"> 1474</span>                    <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_in = h_start + i;</div>
<div class="line"><a id="l01475" name="l01475"></a><span class="lineno"> 1475</span>                    <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_in = w_start + j;</div>
<div class="line"><a id="l01476" name="l01476"></a><span class="lineno"> 1476</span>                    <span class="keywordflow">if</span> (h_in &gt;= 0 &amp;&amp; h_in &lt; H_in &amp;&amp; w_in &gt;= 0 &amp;&amp; w_in &lt; W_in) {</div>
<div class="line"><a id="l01477" name="l01477"></a><span class="lineno"> 1477</span>                        atomicAdd(out + currentBatch * (channels * H_in * W_in) + currentChannel * (H_in * W_in) + h_in * W_in + w_in, in[idx] / (<span class="keywordtype">float</span>)count);</div>
<div class="line"><a id="l01478" name="l01478"></a><span class="lineno"> 1478</span>                    }</div>
<div class="line"><a id="l01479" name="l01479"></a><span class="lineno"> 1479</span>                }</div>
<div class="line"><a id="l01480" name="l01480"></a><span class="lineno"> 1480</span>            }</div>
<div class="line"><a id="l01481" name="l01481"></a><span class="lineno"> 1481</span>        }</div>
<div class="line"><a id="l01482" name="l01482"></a><span class="lineno"> 1482</span>    }</div>
<div class="line"><a id="l01483" name="l01483"></a><span class="lineno"> 1483</span> </div>
<div class="foldopen" id="foldopen01484" data-start="{" data-end="}">
<div class="line"><a id="l01484" name="l01484"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a551402f9c55653c9fae63e172a5fb250"> 1484</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a551402f9c55653c9fae63e172a5fb250">AveragePoolingBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* out, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l01485" name="l01485"></a><span class="lineno"> 1485</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> pool_size, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> padding,</div>
<div class="line"><a id="l01486" name="l01486"></a><span class="lineno"> 1486</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in,</div>
<div class="line"><a id="l01487" name="l01487"></a><span class="lineno"> 1487</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out) {</div>
<div class="line"><a id="l01488" name="l01488"></a><span class="lineno"> 1488</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(AveragePoolingBackwardKernel, gridDim, blockDim, 0, out, in,</div>
<div class="line"><a id="l01489" name="l01489"></a><span class="lineno"> 1489</span>            pool_size, stride, padding, batches, channels, H_in, W_in, H_out, W_out);</div>
<div class="line"><a id="l01490" name="l01490"></a><span class="lineno"> 1490</span>    }</div>
</div>
<div class="line"><a id="l01491" name="l01491"></a><span class="lineno"> 1491</span> </div>
<div class="line"><a id="l01492" name="l01492"></a><span class="lineno"> 1492</span>    __global__ <span class="keywordtype">void</span> GlobalAvgPoolBackwardKernel(<span class="keywordtype">float</span>* output, <span class="keyword">const</span> <span class="keywordtype">float</span>* in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> height, <span class="keyword">const</span> <span class="keywordtype">size_t</span> width) {</div>
<div class="line"><a id="l01493" name="l01493"></a><span class="lineno"> 1493</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01494" name="l01494"></a><span class="lineno"> 1494</span>        <span class="keywordflow">if</span> (idx &gt;= batches * channels * height * width) {</div>
<div class="line"><a id="l01495" name="l01495"></a><span class="lineno"> 1495</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01496" name="l01496"></a><span class="lineno"> 1496</span>        }</div>
<div class="line"><a id="l01497" name="l01497"></a><span class="lineno"> 1497</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentBatch = idx / (channels * height * width);</div>
<div class="line"><a id="l01498" name="l01498"></a><span class="lineno"> 1498</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentChannel = (idx % (channels * height * width)) / (height * width);</div>
<div class="line"><a id="l01499" name="l01499"></a><span class="lineno"> 1499</span>        output[idx] = in[currentBatch * channels + currentChannel] / (float)(height * width);</div>
<div class="line"><a id="l01500" name="l01500"></a><span class="lineno"> 1500</span>    }</div>
<div class="line"><a id="l01501" name="l01501"></a><span class="lineno"> 1501</span> </div>
<div class="foldopen" id="foldopen01502" data-start="{" data-end="}">
<div class="line"><a id="l01502" name="l01502"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a73ceb77688c4008dc350fc87b99875aa"> 1502</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a73ceb77688c4008dc350fc87b99875aa">GlobalAvgPoolBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* output, <span class="keywordtype">float</span>* in,</div>
<div class="line"><a id="l01503" name="l01503"></a><span class="lineno"> 1503</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> height, <span class="keyword">const</span> <span class="keywordtype">size_t</span> width) {</div>
<div class="line"><a id="l01504" name="l01504"></a><span class="lineno"> 1504</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(GlobalAvgPoolBackwardKernel, gridDim, blockDim, 0, output, in,</div>
<div class="line"><a id="l01505" name="l01505"></a><span class="lineno"> 1505</span>            batches, channels, height, width);</div>
<div class="line"><a id="l01506" name="l01506"></a><span class="lineno"> 1506</span>    }</div>
</div>
<div class="line"><a id="l01507" name="l01507"></a><span class="lineno"> 1507</span> </div>
<div class="line"><a id="l01508" name="l01508"></a><span class="lineno"> 1508</span>    __global__ <span class="keywordtype">void</span> MaxPoolingKernel(<span class="keywordtype">float</span>* output, <span class="keywordtype">float</span>* position, <span class="keyword">const</span> <span class="keywordtype">float</span>* input, <span class="keyword">const</span> <span class="keywordtype">size_t</span> pool_size, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> padding,</div>
<div class="line"><a id="l01509" name="l01509"></a><span class="lineno"> 1509</span>    <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out) {</div>
<div class="line"><a id="l01510" name="l01510"></a><span class="lineno"> 1510</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01511" name="l01511"></a><span class="lineno"> 1511</span>        <span class="keywordflow">if</span> (idx &gt;= batches * channels * H_out * W_out) {</div>
<div class="line"><a id="l01512" name="l01512"></a><span class="lineno"> 1512</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01513" name="l01513"></a><span class="lineno"> 1513</span>        }</div>
<div class="line"><a id="l01514" name="l01514"></a><span class="lineno"> 1514</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentBatch = idx / (channels * H_out * W_out);</div>
<div class="line"><a id="l01515" name="l01515"></a><span class="lineno"> 1515</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentChannel = (idx % (channels * H_out * W_out)) / (H_out * W_out);</div>
<div class="line"><a id="l01516" name="l01516"></a><span class="lineno"> 1516</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> h = (idx % (H_out * W_out)) / W_out;</div>
<div class="line"><a id="l01517" name="l01517"></a><span class="lineno"> 1517</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> w = (idx % (H_out * W_out)) % W_out;</div>
<div class="line"><a id="l01518" name="l01518"></a><span class="lineno"> 1518</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_start = h * stride - padding;</div>
<div class="line"><a id="l01519" name="l01519"></a><span class="lineno"> 1519</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_start = w * stride - padding;</div>
<div class="line"><a id="l01520" name="l01520"></a><span class="lineno"> 1520</span>        <span class="keywordtype">float</span> max = -1e20f;</div>
<div class="line"><a id="l01521" name="l01521"></a><span class="lineno"> 1521</span>        <span class="keywordtype">size_t</span> maxIndex = 0;</div>
<div class="line"><a id="l01522" name="l01522"></a><span class="lineno"> 1522</span>        <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> i = 0; i &lt; pool_size; i++) {</div>
<div class="line"><a id="l01523" name="l01523"></a><span class="lineno"> 1523</span>            <span class="keywordflow">for</span> (<span class="keywordtype">long</span> <span class="keywordtype">long</span> j = 0; j &lt; pool_size; j++) {</div>
<div class="line"><a id="l01524" name="l01524"></a><span class="lineno"> 1524</span>                <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_in = h_start + i;</div>
<div class="line"><a id="l01525" name="l01525"></a><span class="lineno"> 1525</span>                <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_in = w_start + j;</div>
<div class="line"><a id="l01526" name="l01526"></a><span class="lineno"> 1526</span>                <span class="keywordflow">if</span> (h_in &gt;= 0 &amp;&amp; h_in &lt; H_in &amp;&amp; w_in &gt;= 0 &amp;&amp; w_in &lt; W_in) {</div>
<div class="line"><a id="l01527" name="l01527"></a><span class="lineno"> 1527</span>                    <span class="keyword">const</span> <span class="keywordtype">float</span> value = input[currentBatch * (channels * H_in * W_in) + currentChannel * (H_in * W_in) + h_in * W_in + w_in];</div>
<div class="line"><a id="l01528" name="l01528"></a><span class="lineno"> 1528</span>                    <span class="keywordflow">if</span> (value &gt; max) {</div>
<div class="line"><a id="l01529" name="l01529"></a><span class="lineno"> 1529</span>                        max = value;</div>
<div class="line"><a id="l01530" name="l01530"></a><span class="lineno"> 1530</span>                        maxIndex = i * pool_size + j;</div>
<div class="line"><a id="l01531" name="l01531"></a><span class="lineno"> 1531</span>                    }</div>
<div class="line"><a id="l01532" name="l01532"></a><span class="lineno"> 1532</span>                }</div>
<div class="line"><a id="l01533" name="l01533"></a><span class="lineno"> 1533</span>            }</div>
<div class="line"><a id="l01534" name="l01534"></a><span class="lineno"> 1534</span>        }</div>
<div class="line"><a id="l01535" name="l01535"></a><span class="lineno"> 1535</span>        output[idx] = max == -1e20f ? 0.0f : max;</div>
<div class="line"><a id="l01536" name="l01536"></a><span class="lineno"> 1536</span>        position[idx] = (float)maxIndex;</div>
<div class="line"><a id="l01537" name="l01537"></a><span class="lineno"> 1537</span>    }</div>
<div class="line"><a id="l01538" name="l01538"></a><span class="lineno"> 1538</span> </div>
<div class="foldopen" id="foldopen01539" data-start="{" data-end="}">
<div class="line"><a id="l01539" name="l01539"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#abcc632e5a7104c1a28208e94a4ce6e28"> 1539</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#abcc632e5a7104c1a28208e94a4ce6e28">MaxPooling</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* output, <span class="keywordtype">float</span>* position, <span class="keywordtype">float</span>* input,</div>
<div class="line"><a id="l01540" name="l01540"></a><span class="lineno"> 1540</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> pool_size, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> padding,</div>
<div class="line"><a id="l01541" name="l01541"></a><span class="lineno"> 1541</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in,</div>
<div class="line"><a id="l01542" name="l01542"></a><span class="lineno"> 1542</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out) {</div>
<div class="line"><a id="l01543" name="l01543"></a><span class="lineno"> 1543</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().submitDualOut(MaxPoolingKernel, gridDim, blockDim, 0, output, position, input,</div>
<div class="line"><a id="l01544" name="l01544"></a><span class="lineno"> 1544</span>            pool_size, stride, padding, batches, channels, H_in, W_in, H_out, W_out);</div>
<div class="line"><a id="l01545" name="l01545"></a><span class="lineno"> 1545</span>    }</div>
</div>
<div class="line"><a id="l01546" name="l01546"></a><span class="lineno"> 1546</span> </div>
<div class="line"><a id="l01547" name="l01547"></a><span class="lineno"> 1547</span>    __global__ <span class="keywordtype">void</span> MaxPoolingBackwardKernel(<span class="keywordtype">float</span>* output, <span class="keyword">const</span> <span class="keywordtype">float</span>* position, <span class="keyword">const</span> <span class="keywordtype">float</span>* input, <span class="keyword">const</span> <span class="keywordtype">size_t</span> pool_size, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> padding,</div>
<div class="line"><a id="l01548" name="l01548"></a><span class="lineno"> 1548</span><span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out) {</div>
<div class="line"><a id="l01549" name="l01549"></a><span class="lineno"> 1549</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> idx = blockIdx.x * blockDim.x + threadIdx.x;</div>
<div class="line"><a id="l01550" name="l01550"></a><span class="lineno"> 1550</span>        <span class="keywordflow">if</span> (idx &gt;= batches * channels * H_out * W_out) {</div>
<div class="line"><a id="l01551" name="l01551"></a><span class="lineno"> 1551</span>            <span class="keywordflow">return</span>;</div>
<div class="line"><a id="l01552" name="l01552"></a><span class="lineno"> 1552</span>        }</div>
<div class="line"><a id="l01553" name="l01553"></a><span class="lineno"> 1553</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentBatch = idx / (channels * H_out * W_out);</div>
<div class="line"><a id="l01554" name="l01554"></a><span class="lineno"> 1554</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> currentChannel = (idx % (channels * H_out * W_out)) / (H_out * W_out);</div>
<div class="line"><a id="l01555" name="l01555"></a><span class="lineno"> 1555</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> h = (idx % (H_out * W_out)) / W_out;</div>
<div class="line"><a id="l01556" name="l01556"></a><span class="lineno"> 1556</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> w = (idx % (H_out * W_out)) % W_out;</div>
<div class="line"><a id="l01557" name="l01557"></a><span class="lineno"> 1557</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_start = h * stride - padding;</div>
<div class="line"><a id="l01558" name="l01558"></a><span class="lineno"> 1558</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_start = w * stride - padding;</div>
<div class="line"><a id="l01559" name="l01559"></a><span class="lineno"> 1559</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> maxIndex = (<span class="keywordtype">long</span> long)position[idx];</div>
<div class="line"><a id="l01560" name="l01560"></a><span class="lineno"> 1560</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> h_in = h_start + maxIndex / pool_size;</div>
<div class="line"><a id="l01561" name="l01561"></a><span class="lineno"> 1561</span>        <span class="keyword">const</span> <span class="keywordtype">long</span> <span class="keywordtype">long</span> w_in = w_start + maxIndex % pool_size;</div>
<div class="line"><a id="l01562" name="l01562"></a><span class="lineno"> 1562</span>        <span class="keywordflow">if</span> (h_in &gt;= 0 &amp;&amp; h_in &lt; H_in &amp;&amp; w_in &gt;= 0 &amp;&amp; w_in &lt; W_in) {</div>
<div class="line"><a id="l01563" name="l01563"></a><span class="lineno"> 1563</span>            atomicAdd(output + currentBatch * (channels * H_in * W_in) + currentChannel * (H_in * W_in) + h_in * W_in + w_in, input[idx]);</div>
<div class="line"><a id="l01564" name="l01564"></a><span class="lineno"> 1564</span>        }</div>
<div class="line"><a id="l01565" name="l01565"></a><span class="lineno"> 1565</span>    }</div>
<div class="line"><a id="l01566" name="l01566"></a><span class="lineno"> 1566</span> </div>
<div class="foldopen" id="foldopen01567" data-start="{" data-end="}">
<div class="line"><a id="l01567" name="l01567"></a><span class="lineno"><a class="line" href="namespacenz_1_1krnl.html#a0d5f5f4c9e89a8d914a7f2f802d1caab"> 1567</a></span>    <span class="keywordtype">void</span> <a class="code hl_function" href="namespacenz_1_1krnl.html#a0d5f5f4c9e89a8d914a7f2f802d1caab">MaxPoolingBackward</a>(<span class="keyword">const</span> dim3 gridDim, <span class="keyword">const</span> dim3 blockDim, <span class="keywordtype">float</span>* output, <span class="keywordtype">float</span>* position, <span class="keywordtype">float</span>* input,</div>
<div class="line"><a id="l01568" name="l01568"></a><span class="lineno"> 1568</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> pool_size, <span class="keyword">const</span> <span class="keywordtype">size_t</span> stride, <span class="keyword">const</span> <span class="keywordtype">size_t</span> padding,</div>
<div class="line"><a id="l01569" name="l01569"></a><span class="lineno"> 1569</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> batches, <span class="keyword">const</span> <span class="keywordtype">size_t</span> channels, <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_in, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_in,</div>
<div class="line"><a id="l01570" name="l01570"></a><span class="lineno"> 1570</span>        <span class="keyword">const</span> <span class="keywordtype">size_t</span> H_out, <span class="keyword">const</span> <span class="keywordtype">size_t</span> W_out) {</div>
<div class="line"><a id="l01571" name="l01571"></a><span class="lineno"> 1571</span>        <a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">StreamManager&lt;float&gt;::Instance</a>().<a class="code hl_function" href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">submit</a>(MaxPoolingBackwardKernel, gridDim, blockDim, 0, output, position, input,</div>
<div class="line"><a id="l01572" name="l01572"></a><span class="lineno"> 1572</span>            pool_size, stride, padding, batches, channels, H_in, W_in, H_out, W_out);</div>
<div class="line"><a id="l01573" name="l01573"></a><span class="lineno"> 1573</span>    }</div>
</div>
<div class="line"><a id="l01574" name="l01574"></a><span class="lineno"> 1574</span>}</div>
<div class="ttc" id="a_operation_kernels_8cuh_html"><div class="ttname"><a href="_operation_kernels_8cuh.html">OperationKernels.cuh</a></div><div class="ttdoc">CUDA Kernel Definitions for High-Performance Tensor Operations.</div></div>
<div class="ttc" id="aclassnz_1_1cu_strm_1_1_stream_manager_html_a1084057ef6f5b2871c60702209bb4469"><div class="ttname"><a href="classnz_1_1cu_strm_1_1_stream_manager.html#a1084057ef6f5b2871c60702209bb4469">nz::cuStrm::StreamManager::freeAsync</a></div><div class="ttdeci">void freeAsync(T *data)</div><div class="ttdoc">Asynchronously frees the CUDA device memory pointed to by the given pointer.</div><div class="ttdef"><b>Definition</b> <a href="_stream_manager_8cuh_source.html#l00325">StreamManager.cuh:325</a></div></div>
<div class="ttc" id="aclassnz_1_1cu_strm_1_1_stream_manager_html_a1260d95d0eddf75b72700da07361a4bd"><div class="ttname"><a href="classnz_1_1cu_strm_1_1_stream_manager.html#a1260d95d0eddf75b72700da07361a4bd">nz::cuStrm::StreamManager::recordData</a></div><div class="ttdeci">void recordData(T *data, cudaStream_t stream)</div><div class="ttdoc">Records write completion event for asynchronous data operations.</div><div class="ttdef"><b>Definition</b> <a href="_stream_manager_8cuh_source.html#l00886">StreamManager.cuh:886</a></div></div>
<div class="ttc" id="aclassnz_1_1cu_strm_1_1_stream_manager_html_a1de1cf3aadea137faf90a2f9b4b7abe2"><div class="ttname"><a href="classnz_1_1cu_strm_1_1_stream_manager.html#a1de1cf3aadea137faf90a2f9b4b7abe2">nz::cuStrm::StreamManager::getStream</a></div><div class="ttdeci">cudaStream_t getStream()</div><div class="ttdoc">Acquires CUDA stream from pool using round-robin scheduling.</div><div class="ttdef"><b>Definition</b> <a href="_stream_manager_8cuh_source.html#l00799">StreamManager.cuh:799</a></div></div>
<div class="ttc" id="aclassnz_1_1cu_strm_1_1_stream_manager_html_a46ce59b45de432842454aadf00b93791"><div class="ttname"><a href="classnz_1_1cu_strm_1_1_stream_manager.html#a46ce59b45de432842454aadf00b93791">nz::cuStrm::StreamManager::submit</a></div><div class="ttdeci">void submit(F func, dim3 grid, dim3 block, size_t shared, T *odata, T *idata, Args... args)</div><div class="ttdoc">Asynchronously submits a CUDA kernel with stream-ordered dependency management.</div><div class="ttdef"><b>Definition</b> <a href="_stream_manager_8cuh_source.html#l00471">StreamManager.cuh:471</a></div></div>
<div class="ttc" id="aclassnz_1_1cu_strm_1_1_stream_manager_html_a71ad766cb2869d3dd6a3931966e81706"><div class="ttname"><a href="classnz_1_1cu_strm_1_1_stream_manager.html#a71ad766cb2869d3dd6a3931966e81706">nz::cuStrm::StreamManager::memset</a></div><div class="ttdeci">void memset(T *data, const int value, const size_t count)</div><div class="ttdoc">Asynchronously sets a block of CUDA device memory to a specified value.</div><div class="ttdef"><b>Definition</b> <a href="_stream_manager_8cuh_source.html#l00360">StreamManager.cuh:360</a></div></div>
<div class="ttc" id="aclassnz_1_1cu_strm_1_1_stream_manager_html_a97f78a2d43f6e0508c82d4f3b629de96"><div class="ttname"><a href="classnz_1_1cu_strm_1_1_stream_manager.html#a97f78a2d43f6e0508c82d4f3b629de96">nz::cuStrm::StreamManager::malloc</a></div><div class="ttdeci">void malloc(T **data, const size_t size)</div><div class="ttdoc">Asynchronously allocates device memory for type-specific data with stream-ordered dependency tracking...</div><div class="ttdef"><b>Definition</b> <a href="_stream_manager_8cuh_source.html#l00230">StreamManager.cuh:230</a></div></div>
<div class="ttc" id="aclassnz_1_1cu_strm_1_1_stream_manager_html_ab4b2eb422e0e1ee44bdfdc0eb94457ce"><div class="ttname"><a href="classnz_1_1cu_strm_1_1_stream_manager.html#ab4b2eb422e0e1ee44bdfdc0eb94457ce">nz::cuStrm::StreamManager::Instance</a></div><div class="ttdeci">static StreamManager &amp; Instance()</div><div class="ttdoc">Returns a reference to the singleton instance of the StreamManager.</div><div class="ttdef"><b>Definition</b> <a href="_stream_manager_8cuh_source.html#l00154">StreamManager.cuh:154</a></div></div>
<div class="ttc" id="aclassnz_1_1cu_strm_1_1_stream_manager_html_adb1078a67c6e38932d7d58c2adb05ec0"><div class="ttname"><a href="classnz_1_1cu_strm_1_1_stream_manager.html#adb1078a67c6e38932d7d58c2adb05ec0">nz::cuStrm::StreamManager::streamWait</a></div><div class="ttdeci">void streamWait(T *data, cudaStream_t stream)</div><div class="ttdoc">Synchronizes CUDA stream execution until data writes complete.</div><div class="ttdef"><b>Definition</b> <a href="_stream_manager_8cuh_source.html#l00840">StreamManager.cuh:840</a></div></div>
<div class="ttc" id="aclassnz_1_1data_1_1_dimension_html"><div class="ttname"><a href="classnz_1_1data_1_1_dimension.html">nz::data::Dimension</a></div><div class="ttdoc">Represents a multi - dimensional shape, typically used in deep learning for tensor dimensions.</div><div class="ttdef"><b>Definition</b> <a href="_dimension_8cuh_source.html#l00057">Dimension.cuh:57</a></div></div>
<div class="ttc" id="aclassnz_1_1data_1_1_dimension_html_a4831fea5aaf7dbad3578d3fa8e55aef1"><div class="ttname"><a href="classnz_1_1data_1_1_dimension.html#a4831fea5aaf7dbad3578d3fa8e55aef1">nz::data::Dimension::getStride</a></div><div class="ttdeci">size_t getStride(size_t i) const</div><div class="ttdoc">Retrieves the stride value at a specified index within the Dimension object.</div><div class="ttdef"><b>Definition</b> <a href="_dimension_8cu_source.html#l00040">Dimension.cu:40</a></div></div>
<div class="ttc" id="aclassnz_1_1data_1_1_dimension_html_a65773c675476dfea3f06b30f21ebbedd"><div class="ttname"><a href="classnz_1_1data_1_1_dimension.html#a65773c675476dfea3f06b30f21ebbedd">nz::data::Dimension::W</a></div><div class="ttdeci">size_t W() const</div><div class="ttdoc">Retrieves the value of the 'w' dimension.</div><div class="ttdef"><b>Definition</b> <a href="_dimension_8cu_source.html#l00063">Dimension.cu:63</a></div></div>
<div class="ttc" id="aclassnz_1_1data_1_1_dimension_html_a7eb3acc882c48e775c418d97f709240f"><div class="ttname"><a href="classnz_1_1data_1_1_dimension.html#a7eb3acc882c48e775c418d97f709240f">nz::data::Dimension::H</a></div><div class="ttdeci">size_t H() const</div><div class="ttdoc">Retrieves the value of the 'h' dimension.</div><div class="ttdef"><b>Definition</b> <a href="_dimension_8cu_source.html#l00059">Dimension.cu:59</a></div></div>
<div class="ttc" id="aclassnz_1_1data_1_1_dimension_html_acc472e84b4c44f649f34b6fbb0eeacf7"><div class="ttname"><a href="classnz_1_1data_1_1_dimension.html#acc472e84b4c44f649f34b6fbb0eeacf7">nz::data::Dimension::N</a></div><div class="ttdeci">size_t N() const</div><div class="ttdoc">Retrieves the value of the 'n' dimension.</div><div class="ttdef"><b>Definition</b> <a href="_dimension_8cu_source.html#l00051">Dimension.cu:51</a></div></div>
<div class="ttc" id="aclassnz_1_1data_1_1_dimension_html_ae1e87c4a462dd60e02821aa27ffc7e09"><div class="ttname"><a href="classnz_1_1data_1_1_dimension.html#ae1e87c4a462dd60e02821aa27ffc7e09">nz::data::Dimension::C</a></div><div class="ttdeci">size_t C() const</div><div class="ttdoc">Retrieves the value of the 'c' dimension.</div><div class="ttdef"><b>Definition</b> <a href="_dimension_8cu_source.html#l00055">Dimension.cu:55</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html"><div class="ttname"><a href="namespacenz_1_1krnl.html">nz::krnl</a></div><div class="ttdoc">High-Performance CUDA Kernel Implementations for Tensor Computations.</div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a028970809074d79f28ff94f62b3edaa4"><div class="ttname"><a href="namespacenz_1_1krnl.html#a028970809074d79f28ff94f62b3edaa4">nz::krnl::col2imgBackward</a></div><div class="ttdeci">void col2imgBackward(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t H_out, size_t W_out, size_t C_out, size_t batches)</div><div class="ttdoc">Rearranges columnar data back into image format for backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l01398">OperationKernels.cu:1398</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a04246c5218530f789a0ed4811b7ef3f3"><div class="ttname"><a href="namespacenz_1_1krnl.html#a04246c5218530f789a0ed4811b7ef3f3">nz::krnl::LeakyReLU</a></div><div class="ttdeci">void LeakyReLU(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n, float alpha=0.01f)</div><div class="ttdoc">Kernel function to apply the Leaky ReLU activation function on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00315">OperationKernels.cu:315</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a0d5f5f4c9e89a8d914a7f2f802d1caab"><div class="ttname"><a href="namespacenz_1_1krnl.html#a0d5f5f4c9e89a8d914a7f2f802d1caab">nz::krnl::MaxPoolingBackward</a></div><div class="ttdeci">void MaxPoolingBackward(dim3 gridDim, dim3 blockDim, float *output, float *position, float *input, size_t pool_size, size_t stride, size_t padding, size_t batches, size_t channels, size_t H_in, size_t W_in, size_t H_out, size_t W_out)</div><div class="ttdoc">Kernel function to compute the gradient of max pooling during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l01567">OperationKernels.cu:1567</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a0e82aca250b46ac8ded8cae8936d7e38"><div class="ttname"><a href="namespacenz_1_1krnl.html#a0e82aca250b46ac8ded8cae8936d7e38">nz::krnl::ExponentialLinearUnit</a></div><div class="ttdeci">void ExponentialLinearUnit(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n, float alpha=1.0f)</div><div class="ttdoc">Kernel function to apply the Exponential Linear Unit (ELU) activation function on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00372">OperationKernels.cu:372</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a0ed44a68bfb86a9fd3d6c3b25614713f"><div class="ttname"><a href="namespacenz_1_1krnl.html#a0ed44a68bfb86a9fd3d6c3b25614713f">nz::krnl::gradCopy</a></div><div class="ttdeci">void gradCopy(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t n, const std::vector&lt; size_t &gt; &amp;offset_o, const std::vector&lt; size_t &gt; &amp;offset_i)</div><div class="ttdoc">Copies gradient data from one array to another with specified offsets.</div><div class="ttdef"><b>Definition</b> <a href="#l01238">OperationKernels.cu:1238</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a1ae846a65c2f5b83cd1b9fc61b877854"><div class="ttname"><a href="namespacenz_1_1krnl.html#a1ae846a65c2f5b83cd1b9fc61b877854">nz::krnl::Summation</a></div><div class="ttdeci">void Summation(dim3 gridDim, dim3 blockDim, unsigned long long sharedMemSize, float *out, float *in, unsigned long long n, size_t offset=0)</div><div class="ttdoc">Kernel function to perform element-wise summation of two arrays.</div><div class="ttdef"><b>Definition</b> <a href="#l01225">OperationKernels.cu:1225</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a1c2b7a6f28d2af22f9a2623c5ae62bff"><div class="ttname"><a href="namespacenz_1_1krnl.html#a1c2b7a6f28d2af22f9a2623c5ae62bff">nz::krnl::img2colBackward</a></div><div class="ttdeci">void img2colBackward(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t H_out, size_t W_out, size_t C, size_t K_h, size_t K_w, size_t stride, size_t pad, size_t H_in, size_t W_in, size_t batch)</div><div class="ttdoc">Rearranges columnar data back into image format for backpropagation in convolution operations.</div><div class="ttdef"><b>Definition</b> <a href="#l01357">OperationKernels.cu:1357</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a1e915bd4a354938d8bc2d09be00eae76"><div class="ttname"><a href="namespacenz_1_1krnl.html#a1e915bd4a354938d8bc2d09be00eae76">nz::krnl::AdaGrad</a></div><div class="ttdeci">void AdaGrad(dim3 gridDim, dim3 blockDim, float *data, float *G, float *grad, float lr, float eps, unsigned long long n)</div><div class="ttdoc">Kernel function to apply AdaGrad optimization.</div><div class="ttdef"><b>Definition</b> <a href="#l00731">OperationKernels.cu:731</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a1f71726879c2d6a9d790522cdc1576e1"><div class="ttname"><a href="namespacenz_1_1krnl.html#a1f71726879c2d6a9d790522cdc1576e1">nz::krnl::AdaDelta</a></div><div class="ttdeci">void AdaDelta(dim3 gridDim, dim3 blockDim, float *data, float *acc_delta, float *acc_grad, float *grad, float rho, float eps, unsigned long long n)</div><div class="ttdoc">Kernel function to apply AdaDelta optimization.</div><div class="ttdef"><b>Definition</b> <a href="#l00815">OperationKernels.cu:815</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a1fc3d553947a5cad87f29989f9d9465d"><div class="ttname"><a href="namespacenz_1_1krnl.html#a1fc3d553947a5cad87f29989f9d9465d">nz::krnl::BCEBackward</a></div><div class="ttdeci">void BCEBackward(dim3 gridDim, dim3 blockDim, float *out, float *predict, float *real, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the gradient of Binary Cross Entropy (BCE) loss for backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00701">OperationKernels.cu:701</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a21bbbcf6d97bfaccc828ce7736814bd4"><div class="ttname"><a href="namespacenz_1_1krnl.html#a21bbbcf6d97bfaccc828ce7736814bd4">nz::krnl::Sigmoid</a></div><div class="ttdeci">void Sigmoid(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n)</div><div class="ttdoc">Kernel function to apply the Sigmoid activation function on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00263">OperationKernels.cu:263</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a273ef3023442a864f1028becaf236bae"><div class="ttname"><a href="namespacenz_1_1krnl.html#a273ef3023442a864f1028becaf236bae">nz::krnl::Momentum</a></div><div class="ttdeci">void Momentum(dim3 gridDim, dim3 blockDim, float *output, float *grad, float *velocity, float beta, unsigned long long n)</div><div class="ttdoc">Kernel function to apply Momentum optimization.</div><div class="ttdef"><b>Definition</b> <a href="#l00715">OperationKernels.cu:715</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a27bc4025be4253d5fffae2bf1b43b3af"><div class="ttname"><a href="namespacenz_1_1krnl.html#a27bc4025be4253d5fffae2bf1b43b3af">nz::krnl::ScalarDiv</a></div><div class="ttdeci">void ScalarDiv(dim3 gridDim, dim3 blockDim, float *out, float *in, float num, unsigned long long n)</div><div class="ttdoc">Kernel function to perform scalar division on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00183">OperationKernels.cu:183</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a2b9ab840eeb0e74f4b78277a046b3a07"><div class="ttname"><a href="namespacenz_1_1krnl.html#a2b9ab840eeb0e74f4b78277a046b3a07">nz::krnl::Adam</a></div><div class="ttdeci">void Adam(dim3 gridDim, dim3 blockDim, float *data, float *m, float *v, float *grad, float lr, float beta1, float beta2, float eps, int t, unsigned long long n)</div><div class="ttdoc">Kernel function to apply Adam optimization.</div><div class="ttdef"><b>Definition</b> <a href="#l00768">OperationKernels.cu:768</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a3a781324400c54c35dd564f3599dca8e"><div class="ttname"><a href="namespacenz_1_1krnl.html#a3a781324400c54c35dd564f3599dca8e">nz::krnl::img2col</a></div><div class="ttdeci">void img2col(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t H_out, size_t W_out, size_t C, size_t K_h, size_t K_w, size_t stride, size_t pad, size_t H_in, size_t W_in, size_t batch)</div><div class="ttdoc">Rearranges image data into column format for convolution operations.</div><div class="ttdef"><b>Definition</b> <a href="#l01330">OperationKernels.cu:1330</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a43232f9472ad3b974351e59386208efa"><div class="ttname"><a href="namespacenz_1_1krnl.html#a43232f9472ad3b974351e59386208efa">nz::krnl::HardSigmoidBackward</a></div><div class="ttdeci">void HardSigmoidBackward(dim3 gridDim, dim3 blockDim, float *A_grad, float *A, float *B_grad, unsigned long long n, float alpha=0.2f, float beta=0.5f)</div><div class="ttdoc">Kernel function to compute the gradient of the Hard Sigmoid activation during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00424">OperationKernels.cu:424</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a4375738c83ef892783abc210578e5b39"><div class="ttname"><a href="namespacenz_1_1krnl.html#a4375738c83ef892783abc210578e5b39">nz::krnl::SoftmaxJacobian</a></div><div class="ttdeci">void SoftmaxJacobian(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the Jacobian of the Softmax function.</div><div class="ttdef"><b>Definition</b> <a href="#l00567">OperationKernels.cu:567</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a454a28ef0e22014efca1ede4e954db65"><div class="ttname"><a href="namespacenz_1_1krnl.html#a454a28ef0e22014efca1ede4e954db65">nz::krnl::Compress</a></div><div class="ttdeci">void Compress(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t n, size_t total)</div><div class="ttdoc">Compresses the input array into the output array with a specified total size.</div><div class="ttdef"><b>Definition</b> <a href="#l01303">OperationKernels.cu:1303</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a455365870d43ff26687a731d15c4cdff"><div class="ttname"><a href="namespacenz_1_1krnl.html#a455365870d43ff26687a731d15c4cdff">nz::krnl::HardSwishBackward</a></div><div class="ttdeci">void HardSwishBackward(dim3 gridDim, dim3 blockDim, float *A_grad, float *A, float *B_grad, unsigned long long n, float alpha=0.2f, float beta=0.5f)</div><div class="ttdoc">Kernel function to compute the gradient of the Hard Swish activation during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00462">OperationKernels.cu:462</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a4ddfc808de99fe831e74a3bd3f9bbdaf"><div class="ttname"><a href="namespacenz_1_1krnl.html#a4ddfc808de99fe831e74a3bd3f9bbdaf">nz::krnl::ReLUBackward</a></div><div class="ttdeci">void ReLUBackward(dim3 gridDim, dim3 blockDim, float *A_grad, float *A, float *B_grad, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the gradient of the ReLU activation during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00250">OperationKernels.cu:250</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a51a5ff3c8cc2c3051fddf32de294b467"><div class="ttname"><a href="namespacenz_1_1krnl.html#a51a5ff3c8cc2c3051fddf32de294b467">nz::krnl::SummationExp</a></div><div class="ttdeci">void SummationExp(dim3 gridDim, dim3 blockDim, size_t sharedMemSize, float *out, float *g_data, unsigned long long n, size_t offset=0)</div><div class="ttdoc">Kernel function to compute the summation of exponentials of each element in the input array.</div><div class="ttdef"><b>Definition</b> <a href="#l00510">OperationKernels.cu:510</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a52e449285e560185378234aecaf2f87c"><div class="ttname"><a href="namespacenz_1_1krnl.html#a52e449285e560185378234aecaf2f87c">nz::krnl::HardSigmoid</a></div><div class="ttdeci">void HardSigmoid(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n, float alpha=0.2f, float beta=0.5f)</div><div class="ttdoc">Kernel function to apply the Hard Sigmoid activation function on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00403">OperationKernels.cu:403</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a551402f9c55653c9fae63e172a5fb250"><div class="ttname"><a href="namespacenz_1_1krnl.html#a551402f9c55653c9fae63e172a5fb250">nz::krnl::AveragePoolingBackward</a></div><div class="ttdeci">void AveragePoolingBackward(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t pool_size, size_t stride, size_t padding, size_t batches, size_t channels, size_t H_in, size_t W_in, size_t H_out, size_t W_out)</div><div class="ttdoc">Kernel function to compute the gradient of average pooling during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l01484">OperationKernels.cu:1484</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a56f84e531825be8b2b0974c2488eb765"><div class="ttname"><a href="namespacenz_1_1krnl.html#a56f84e531825be8b2b0974c2488eb765">nz::krnl::ScalarAdd</a></div><div class="ttdeci">void ScalarAdd(dim3 gridDim, dim3 blockDim, float *out, float *in, float num, unsigned long long n)</div><div class="ttdoc">Kernel function to add a scalar to each element of a matrix on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00196">OperationKernels.cu:196</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a5af716524e248c61f3dce227d8ef6e34"><div class="ttname"><a href="namespacenz_1_1krnl.html#a5af716524e248c61f3dce227d8ef6e34">nz::krnl::ScalarMul</a></div><div class="ttdeci">void ScalarMul(dim3 gridDim, dim3 blockDim, float *out, float *in, float num, unsigned long long n)</div><div class="ttdoc">Kernel function to perform scalar multiplication on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00170">OperationKernels.cu:170</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a6c5a4b54442aab42df5afe8688e71596"><div class="ttname"><a href="namespacenz_1_1krnl.html#a6c5a4b54442aab42df5afe8688e71596">nz::krnl::SwishBackward</a></div><div class="ttdeci">void SwishBackward(dim3 gridDim, dim3 blockDim, float *A_grad, float *A, float *B, float *B_grad, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the gradient of the Swish activation during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00359">OperationKernels.cu:359</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a73ceb77688c4008dc350fc87b99875aa"><div class="ttname"><a href="namespacenz_1_1krnl.html#a73ceb77688c4008dc350fc87b99875aa">nz::krnl::GlobalAvgPoolBackward</a></div><div class="ttdeci">void GlobalAvgPoolBackward(dim3 gridDim, dim3 blockDim, float *output, float *in, size_t batches, size_t channels, size_t height, size_t width)</div><div class="ttdoc">Kernel function to compute the gradient of global average pooling during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l01502">OperationKernels.cu:1502</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a7c061f5511c3ab9d36563757bd969ff7"><div class="ttname"><a href="namespacenz_1_1krnl.html#a7c061f5511c3ab9d36563757bd969ff7">nz::krnl::col2img</a></div><div class="ttdeci">void col2img(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t H_out, size_t W_out, size_t C_out, size_t batches)</div><div class="ttdoc">Rearranges columnar data back into image format.</div><div class="ttdef"><b>Definition</b> <a href="#l01378">OperationKernels.cu:1378</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a7eade95ddcf48141d69bb19803b22d51"><div class="ttname"><a href="namespacenz_1_1krnl.html#a7eade95ddcf48141d69bb19803b22d51">nz::krnl::LeakyReLUBackward</a></div><div class="ttdeci">void LeakyReLUBackward(dim3 gridDim, dim3 blockDim, float *A_grad, float *A, float *B_grad, unsigned long long n, float alpha=0.01f)</div><div class="ttdoc">Kernel function to compute the gradient of the Leaky ReLU activation during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00330">OperationKernels.cu:330</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a8855f411733f7de29d013f4ad40096c9"><div class="ttname"><a href="namespacenz_1_1krnl.html#a8855f411733f7de29d013f4ad40096c9">nz::krnl::RectifiedLinearUnit</a></div><div class="ttdeci">void RectifiedLinearUnit(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n)</div><div class="ttdoc">Kernel function to apply the Rectified Linear Unit (ReLU) activation on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00237">OperationKernels.cu:237</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a8ec4524fdefd3d771c72e77e94281c88"><div class="ttname"><a href="namespacenz_1_1krnl.html#a8ec4524fdefd3d771c72e77e94281c88">nz::krnl::HadamardProduct</a></div><div class="ttdeci">void HadamardProduct(dim3 gridDim, dim3 blockDim, float *out, float *in1, float *in2, unsigned long long n)</div><div class="ttdoc">Kernel function to perform element-wise Hadamard product of two arrays.</div><div class="ttdef"><b>Definition</b> <a href="#l01165">OperationKernels.cu:1165</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a90d501e72361b7341f36394af0f27c74"><div class="ttname"><a href="namespacenz_1_1krnl.html#a90d501e72361b7341f36394af0f27c74">nz::krnl::TanhBackward</a></div><div class="ttdeci">void TanhBackward(dim3 gridDim, dim3 blockDim, float *A_grad, float *B, float *B_grad, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the gradient of the Tanh activation during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00302">OperationKernels.cu:302</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a97cda6dfc6545efaee2b686eed9ae766"><div class="ttname"><a href="namespacenz_1_1krnl.html#a97cda6dfc6545efaee2b686eed9ae766">nz::krnl::MatrixAdd</a></div><div class="ttdeci">void MatrixAdd(dim3 gridDim, dim3 blockDim, float *a, float *b, float *c, unsigned long long n, size_t offset_c=0, size_t offset_a=0, size_t offset_b=0)</div><div class="ttdoc">Kernel function to perform matrix addition on GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00026">OperationKernels.cu:26</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a997aa5460fd64fadf9b701fbf73e3fb2"><div class="ttname"><a href="namespacenz_1_1krnl.html#a997aa5460fd64fadf9b701fbf73e3fb2">nz::krnl::Swish</a></div><div class="ttdeci">void Swish(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n)</div><div class="ttdoc">Kernel function to apply the Swish activation function on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00344">OperationKernels.cu:344</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_a9ac0590fbb5eb7f51b05da574e9845a8"><div class="ttname"><a href="namespacenz_1_1krnl.html#a9ac0590fbb5eb7f51b05da574e9845a8">nz::krnl::NgradCopy</a></div><div class="ttdeci">void NgradCopy(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t n, const std::vector&lt; size_t &gt; &amp;offset_o, const std::vector&lt; size_t &gt; &amp;offset_i)</div><div class="ttdoc">Copies gradient data from one array to another with specified offsets.</div><div class="ttdef"><b>Definition</b> <a href="#l01264">OperationKernels.cu:1264</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_aa61cded4977bb2dc3720f7057cc2fb47"><div class="ttname"><a href="namespacenz_1_1krnl.html#aa61cded4977bb2dc3720f7057cc2fb47">nz::krnl::ElementwiseDivide</a></div><div class="ttdeci">void ElementwiseDivide(dim3 gridDim, dim3 blockDim, float *out, float *in1, float *in2, unsigned long long n, size_t offset_o=0, size_t offset_1=0, size_t offset_2=0)</div><div class="ttdoc">Kernel function to perform element-wise division of two arrays.</div><div class="ttdef"><b>Definition</b> <a href="#l01181">OperationKernels.cu:1181</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_aa84aa2397f4f5a09a96bef76726e46f0"><div class="ttname"><a href="namespacenz_1_1krnl.html#aa84aa2397f4f5a09a96bef76726e46f0">nz::krnl::TensorCoreGEMM</a></div><div class="ttdeci">void TensorCoreGEMM(float *A, float *B, float *C, unsigned long long M, unsigned long long N, unsigned long long K)</div><div class="ttdoc">Kernel function to perform fast matrix multiplication using Tensor Cores with half-precision (FP16) s...</div><div class="ttdef"><b>Definition</b> <a href="#l00885">OperationKernels.cu:885</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_aaf3c9cca114d003130ffa4354b4a24de"><div class="ttname"><a href="namespacenz_1_1krnl.html#aaf3c9cca114d003130ffa4354b4a24de">nz::krnl::RMSprop</a></div><div class="ttdeci">void RMSprop(dim3 gridDim, dim3 blockDim, float *data, float *v, float *grad, float lr, float beta, float eps, unsigned long long n)</div><div class="ttdoc">Kernel function to apply RMSprop optimization.</div><div class="ttdef"><b>Definition</b> <a href="#l00747">OperationKernels.cu:747</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_abcc632e5a7104c1a28208e94a4ce6e28"><div class="ttname"><a href="namespacenz_1_1krnl.html#abcc632e5a7104c1a28208e94a4ce6e28">nz::krnl::MaxPooling</a></div><div class="ttdeci">void MaxPooling(dim3 gridDim, dim3 blockDim, float *output, float *position, float *input, size_t pool_size, size_t stride, size_t padding, size_t batches, size_t channels, size_t H_in, size_t W_in, size_t H_out, size_t W_out)</div><div class="ttdoc">Kernel function to perform max pooling on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l01539">OperationKernels.cu:1539</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_abf927faf0950fbc215564c67b8ac57be"><div class="ttname"><a href="namespacenz_1_1krnl.html#abf927faf0950fbc215564c67b8ac57be">nz::krnl::BinaryCrossEntropy</a></div><div class="ttdeci">void BinaryCrossEntropy(dim3 gridDim, dim3 blockDim, size_t sharedMemSize, float *out, float *predict, float *real, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the Binary Cross Entropy (BCE) loss between predicted and real values.</div><div class="ttdef"><b>Definition</b> <a href="#l00686">OperationKernels.cu:686</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_ad136c8a6560a5305984ce0a31bea71bf"><div class="ttname"><a href="namespacenz_1_1krnl.html#ad136c8a6560a5305984ce0a31bea71bf">nz::krnl::Fill</a></div><div class="ttdeci">void Fill(dim3 gridDim, dim3 blockDim, float *data, float value, unsigned long long n, size_t offset=0)</div><div class="ttdoc">Kernel function to fill a data array with a given value.</div><div class="ttdef"><b>Definition</b> <a href="#l01153">OperationKernels.cu:1153</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_ad18a2b0efc0cdfc9cb861396ad4da53f"><div class="ttname"><a href="namespacenz_1_1krnl.html#ad18a2b0efc0cdfc9cb861396ad4da53f">nz::krnl::MatrixSub</a></div><div class="ttdeci">void MatrixSub(dim3 gridDim, dim3 blockDim, float *a, float *b, float *c, unsigned long long n, size_t offset_c=0, size_t offset_a=0, size_t offset_b=0)</div><div class="ttdoc">Kernel function to perform matrix subtraction on GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00050">OperationKernels.cu:50</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_ada94b8c5c6e6d72132face63a3305624"><div class="ttname"><a href="namespacenz_1_1krnl.html#ada94b8c5c6e6d72132face63a3305624">nz::krnl::NAdam</a></div><div class="ttdeci">void NAdam(dim3 gridDim, dim3 blockDim, float *data, float *m, float *m_modified, float *v, float *grad, float lr, float beta1, float beta2, float eps, int t, unsigned long long n)</div><div class="ttdoc">Kernel function to apply NAdam optimization.</div><div class="ttdef"><b>Definition</b> <a href="#l00793">OperationKernels.cu:793</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_adbafc409d57fa0a9d78ecac5bf7b10a3"><div class="ttname"><a href="namespacenz_1_1krnl.html#adbafc409d57fa0a9d78ecac5bf7b10a3">nz::krnl::Softmax</a></div><div class="ttdeci">void Softmax(dim3 gridDim, dim3 blockDim, float *out, float *in, float exp_sum_of_input, unsigned long long n, size_t offset=0)</div><div class="ttdoc">Kernel function to apply the Softmax function on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00525">OperationKernels.cu:525</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_adc047e65307dbc711235f637227b7d10"><div class="ttname"><a href="namespacenz_1_1krnl.html#adc047e65307dbc711235f637227b7d10">nz::krnl::Recip</a></div><div class="ttdeci">void Recip(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the reciprocal of each element of a matrix on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00226">OperationKernels.cu:226</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_addaa377a94d007df2690043b08904e28"><div class="ttname"><a href="namespacenz_1_1krnl.html#addaa377a94d007df2690043b08904e28">nz::krnl::AveragePooling</a></div><div class="ttdeci">void AveragePooling(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t pool_size, size_t stride, size_t padding, size_t batches, size_t channels, size_t H_in, size_t W_in, size_t H_out, size_t W_out)</div><div class="ttdoc">Kernel function to perform average pooling on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l01431">OperationKernels.cu:1431</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_ae30a6e1de69588aa0c6eb8a5b8e6e826"><div class="ttname"><a href="namespacenz_1_1krnl.html#ae30a6e1de69588aa0c6eb8a5b8e6e826">nz::krnl::GeneralMatrixMul</a></div><div class="ttdeci">void GeneralMatrixMul(dim3 gridDim, dim3 blockDim, float *A, float *B, float *C, unsigned long long M, unsigned long long N, unsigned long long K, size_t offset_c=0, size_t offset_a=0, size_t offset_b=0)</div><div class="ttdoc">Kernel function to perform single-precision matrix multiplication on GPU using CUDA cores.</div><div class="ttdef"><b>Definition</b> <a href="#l00103">OperationKernels.cu:103</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_ae45dbebceb76ddf82fa5e6b9df882e62"><div class="ttname"><a href="namespacenz_1_1krnl.html#ae45dbebceb76ddf82fa5e6b9df882e62">nz::krnl::Expand</a></div><div class="ttdeci">void Expand(dim3 gridDim, dim3 blockDim, float *out, float *in, size_t n, size_t total)</div><div class="ttdoc">Expands the input array into the output array with a specified total size.</div><div class="ttdef"><b>Definition</b> <a href="#l01290">OperationKernels.cu:1290</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_ae77920db6adf79a17dbfb1dbf1ab5656"><div class="ttname"><a href="namespacenz_1_1krnl.html#ae77920db6adf79a17dbfb1dbf1ab5656">nz::krnl::MSEBackward</a></div><div class="ttdeci">void MSEBackward(dim3 gridDim, dim3 blockDim, float *out, float *predict, float *real, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the gradient of the Mean Squared Error (MSE) loss for backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00629">OperationKernels.cu:629</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_aeb7d10939b25508e0b5db1fe44f4b467"><div class="ttname"><a href="namespacenz_1_1krnl.html#aeb7d10939b25508e0b5db1fe44f4b467">nz::krnl::Tanh</a></div><div class="ttdeci">void Tanh(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n)</div><div class="ttdoc">Kernel function to apply the Tanh activation function on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00289">OperationKernels.cu:289</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_aee8ca471aa260bd1fca5b1797e229f9f"><div class="ttname"><a href="namespacenz_1_1krnl.html#aee8ca471aa260bd1fca5b1797e229f9f">nz::krnl::ELUBackward</a></div><div class="ttdeci">void ELUBackward(dim3 gridDim, dim3 blockDim, float *A_grad, float *A, float *B_grad, unsigned long long n, float alpha=1.0f)</div><div class="ttdoc">Kernel function to compute the gradient of the ELU activation during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00388">OperationKernels.cu:388</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_aeec286d5351eee7061e151470adb4eef"><div class="ttname"><a href="namespacenz_1_1krnl.html#aeec286d5351eee7061e151470adb4eef">nz::krnl::StochasticGradientDescent</a></div><div class="ttdeci">void StochasticGradientDescent(dim3 gridDim, dim3 blockDim, float *data, float *grad, float lr, unsigned long long n)</div><div class="ttdoc">Kernel function to perform Stochastic Gradient Descent (SGD) optimization.</div><div class="ttdef"><b>Definition</b> <a href="#l00642">OperationKernels.cu:642</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_aef9c028ed356b5684e103639bb23bcf0"><div class="ttname"><a href="namespacenz_1_1krnl.html#aef9c028ed356b5684e103639bb23bcf0">nz::krnl::HardSwish</a></div><div class="ttdeci">void HardSwish(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n, float alpha=0.2f, float beta=0.5f)</div><div class="ttdoc">Kernel function to apply the Hard Swish activation function on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00445">OperationKernels.cu:445</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_af7069a420e81babb49b1bc009333d053"><div class="ttname"><a href="namespacenz_1_1krnl.html#af7069a420e81babb49b1bc009333d053">nz::krnl::Negation</a></div><div class="ttdeci">void Negation(dim3 gridDim, dim3 blockDim, float *out, float *in, unsigned long long n)</div><div class="ttdoc">Kernel function to negate each element of a matrix on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00209">OperationKernels.cu:209</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_af76ce6a930db4def5ceb51350af72f3c"><div class="ttname"><a href="namespacenz_1_1krnl.html#af76ce6a930db4def5ceb51350af72f3c">nz::krnl::MeanSquaredError</a></div><div class="ttdeci">void MeanSquaredError(dim3 gridDim, dim3 blockDim, size_t sharedMemSize, float *out, float *predict, float *real, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the Mean Squared Error (MSE) loss between predicted and real values.</div><div class="ttdef"><b>Definition</b> <a href="#l00615">OperationKernels.cu:615</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_afe3f38f788c735b7eb718443eb0fd094"><div class="ttname"><a href="namespacenz_1_1krnl.html#afe3f38f788c735b7eb718443eb0fd094">nz::krnl::Transpose</a></div><div class="ttdeci">void Transpose(dim3 gridDim, dim3 blockDim, float *d_A, float *d_B, unsigned int rows, unsigned int cols, size_t offset=0)</div><div class="ttdoc">Kernel function to transpose a matrix on the GPU.</div><div class="ttdef"><b>Definition</b> <a href="#l00147">OperationKernels.cu:147</a></div></div>
<div class="ttc" id="anamespacenz_1_1krnl_html_aff1f9f1bf9fb677024bd2b565fab9801"><div class="ttname"><a href="namespacenz_1_1krnl.html#aff1f9f1bf9fb677024bd2b565fab9801">nz::krnl::SigmoidBackward</a></div><div class="ttdeci">void SigmoidBackward(dim3 gridDim, dim3 blockDim, float *A_grad, float *B, float *B_grad, unsigned long long n)</div><div class="ttdoc">Kernel function to compute the gradient of the Sigmoid activation during backpropagation.</div><div class="ttdef"><b>Definition</b> <a href="#l00277">OperationKernels.cu:277</a></div></div>
</div><!-- fragment --></div><!-- contents -->
<!-- start footer part -->
<hr class="footer"/><address class="footer"><small>
Generated by&#160;<a href="https://www.doxygen.org/index.html"><img class="footer" src="doxygen.svg" width="104" height="31" alt="doxygen"/></a> 1.12.0
</small></address>
</div><!-- doc-content -->
</body>
</html>
